2024-11-12 00:52:40,863 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-12 00:52:40,876 main DEBUG Took 0.011297 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 00:52:40,877 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 00:52:40,877 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 00:52:40,878 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 00:52:40,879 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 00:52:40,887 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 00:52:40,901 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 00:52:40,903 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 00:52:40,904 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 00:52:40,904 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 00:52:40,905 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 00:52:40,905 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 00:52:40,907 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 00:52:40,907 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 00:52:40,908 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 00:52:40,908 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 00:52:40,909 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 00:52:40,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 00:52:40,911 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 00:52:40,911 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 00:52:40,912 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 00:52:40,912 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 00:52:40,913 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 00:52:40,913 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 00:52:40,914 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 00:52:40,914 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 00:52:40,915 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 00:52:40,915 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 00:52:40,916 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 00:52:40,916 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 00:52:40,917 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 00:52:40,917 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 00:52:40,919 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 00:52:40,921 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 00:52:40,924 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 00:52:40,924 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 00:52:40,926 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 00:52:40,927 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 00:52:40,937 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 00:52:40,939 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 00:52:40,941 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 00:52:40,942 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 00:52:40,942 main DEBUG createAppenders(={Console}) 2024-11-12 00:52:40,943 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-12 00:52:40,944 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-12 00:52:40,944 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-12 00:52:40,945 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 00:52:40,945 main DEBUG OutputStream closed 2024-11-12 00:52:40,946 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 00:52:40,946 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 00:52:40,947 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-12 00:52:41,013 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 00:52:41,015 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 00:52:41,016 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 00:52:41,017 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 00:52:41,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 00:52:41,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 00:52:41,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 00:52:41,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 00:52:41,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 00:52:41,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 00:52:41,019 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 00:52:41,020 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 00:52:41,020 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 00:52:41,020 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 00:52:41,021 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 00:52:41,021 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 00:52:41,021 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 00:52:41,022 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 00:52:41,024 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 00:52:41,024 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-12 00:52:41,025 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 00:52:41,025 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-12T00:52:41,234 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885 2024-11-12 00:52:41,237 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 00:52:41,237 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T00:52:41,245 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-12T00:52:41,265 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T00:52:41,280 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/cluster_7cc88e89-ee61-c4b1-2bfb-37ef9ae21913, deleteOnExit=true 2024-11-12T00:52:41,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-12T00:52:41,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/test.cache.data in system properties and HBase conf 2024-11-12T00:52:41,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T00:52:41,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/hadoop.log.dir in system properties and HBase conf 2024-11-12T00:52:41,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T00:52:41,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T00:52:41,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T00:52:41,375 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T00:52:41,492 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T00:52:41,497 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T00:52:41,497 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T00:52:41,498 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T00:52:41,499 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T00:52:41,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T00:52:41,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T00:52:41,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T00:52:41,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T00:52:41,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T00:52:41,502 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/nfs.dump.dir in system properties and HBase conf 2024-11-12T00:52:41,502 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/java.io.tmpdir in system properties and HBase conf 2024-11-12T00:52:41,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T00:52:41,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T00:52:41,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T00:52:42,420 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T00:52:42,528 INFO [Time-limited test {}] log.Log(170): Logging initialized @2403ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T00:52:42,613 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T00:52:42,679 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T00:52:42,698 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T00:52:42,698 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T00:52:42,699 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T00:52:42,712 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T00:52:42,716 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/hadoop.log.dir/,AVAILABLE} 2024-11-12T00:52:42,717 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T00:52:42,892 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/java.io.tmpdir/jetty-localhost-35171-hadoop-hdfs-3_4_1-tests_jar-_-any-11552543074232642744/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T00:52:42,898 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:35171} 2024-11-12T00:52:42,898 INFO [Time-limited test {}] server.Server(415): Started @2775ms 2024-11-12T00:52:43,274 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T00:52:43,281 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T00:52:43,282 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T00:52:43,283 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T00:52:43,283 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T00:52:43,284 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/hadoop.log.dir/,AVAILABLE} 2024-11-12T00:52:43,285 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T00:52:43,388 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/java.io.tmpdir/jetty-localhost-46543-hadoop-hdfs-3_4_1-tests_jar-_-any-13986493248528714986/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T00:52:43,388 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:46543} 2024-11-12T00:52:43,389 INFO [Time-limited test {}] server.Server(415): Started @3266ms 2024-11-12T00:52:43,446 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T00:52:43,875 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/cluster_7cc88e89-ee61-c4b1-2bfb-37ef9ae21913/data/data2/current/BP-2013931983-172.17.0.2-1731372762199/current, will proceed with Du for space computation calculation, 2024-11-12T00:52:43,875 WARN [Thread-70 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/cluster_7cc88e89-ee61-c4b1-2bfb-37ef9ae21913/data/data1/current/BP-2013931983-172.17.0.2-1731372762199/current, will proceed with Du for space computation calculation, 2024-11-12T00:52:43,914 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T00:52:43,967 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x552199a757ae1c3c with lease ID 0x3b6a514915e54609: Processing first storage report for DS-04b1c5dc-6d16-42ad-9f87-e9415d4ad469 from datanode DatanodeRegistration(127.0.0.1:40185, datanodeUuid=05c39f0c-41e5-44bb-b94b-55d15cfa9797, infoPort=39725, infoSecurePort=0, ipcPort=46791, storageInfo=lv=-57;cid=testClusterID;nsid=381458333;c=1731372762199) 2024-11-12T00:52:43,968 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x552199a757ae1c3c with lease ID 0x3b6a514915e54609: from storage DS-04b1c5dc-6d16-42ad-9f87-e9415d4ad469 node DatanodeRegistration(127.0.0.1:40185, datanodeUuid=05c39f0c-41e5-44bb-b94b-55d15cfa9797, infoPort=39725, infoSecurePort=0, ipcPort=46791, storageInfo=lv=-57;cid=testClusterID;nsid=381458333;c=1731372762199), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T00:52:43,969 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x552199a757ae1c3c with lease ID 0x3b6a514915e54609: Processing first storage report for DS-88b993bf-3ad7-485c-a561-93f3f29aa6d5 from datanode DatanodeRegistration(127.0.0.1:40185, datanodeUuid=05c39f0c-41e5-44bb-b94b-55d15cfa9797, infoPort=39725, infoSecurePort=0, ipcPort=46791, storageInfo=lv=-57;cid=testClusterID;nsid=381458333;c=1731372762199) 2024-11-12T00:52:43,969 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x552199a757ae1c3c with lease ID 0x3b6a514915e54609: from storage DS-88b993bf-3ad7-485c-a561-93f3f29aa6d5 node DatanodeRegistration(127.0.0.1:40185, datanodeUuid=05c39f0c-41e5-44bb-b94b-55d15cfa9797, infoPort=39725, infoSecurePort=0, ipcPort=46791, storageInfo=lv=-57;cid=testClusterID;nsid=381458333;c=1731372762199), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T00:52:44,058 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885 2024-11-12T00:52:44,125 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/cluster_7cc88e89-ee61-c4b1-2bfb-37ef9ae21913/zookeeper_0, clientPort=64666, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/cluster_7cc88e89-ee61-c4b1-2bfb-37ef9ae21913/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/cluster_7cc88e89-ee61-c4b1-2bfb-37ef9ae21913/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T00:52:44,134 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64666 2024-11-12T00:52:44,150 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T00:52:44,154 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T00:52:44,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741825_1001 (size=7) 2024-11-12T00:52:44,791 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6 with version=8 2024-11-12T00:52:44,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/hbase-staging 2024-11-12T00:52:44,896 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T00:52:45,096 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6928b69dca1b:0 server-side Connection retries=45 2024-11-12T00:52:45,105 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T00:52:45,105 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T00:52:45,110 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T00:52:45,110 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T00:52:45,110 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T00:52:45,259 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T00:52:45,315 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T00:52:45,327 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T00:52:45,332 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T00:52:45,357 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 42228 (auto-detected) 2024-11-12T00:52:45,358 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-12T00:52:45,375 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45925 2024-11-12T00:52:45,395 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45925 connecting to ZooKeeper ensemble=127.0.0.1:64666 2024-11-12T00:52:45,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:459250x0, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T00:52:45,423 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45925-0x100316b283a0000 connected 2024-11-12T00:52:45,448 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T00:52:45,451 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T00:52:45,461 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T00:52:45,465 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6, hbase.cluster.distributed=false 2024-11-12T00:52:45,486 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T00:52:45,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45925 2024-11-12T00:52:45,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45925 2024-11-12T00:52:45,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45925 2024-11-12T00:52:45,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45925 2024-11-12T00:52:45,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45925 2024-11-12T00:52:45,590 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6928b69dca1b:0 server-side Connection retries=45 2024-11-12T00:52:45,591 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T00:52:45,591 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T00:52:45,592 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T00:52:45,592 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T00:52:45,592 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T00:52:45,595 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T00:52:45,599 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T00:52:45,600 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33953 2024-11-12T00:52:45,602 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33953 connecting to ZooKeeper ensemble=127.0.0.1:64666 2024-11-12T00:52:45,603 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T00:52:45,606 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T00:52:45,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:339530x0, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T00:52:45,614 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33953-0x100316b283a0001 connected 2024-11-12T00:52:45,614 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T00:52:45,619 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T00:52:45,628 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T00:52:45,631 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T00:52:45,636 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T00:52:45,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33953 2024-11-12T00:52:45,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33953 2024-11-12T00:52:45,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33953 2024-11-12T00:52:45,639 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33953 2024-11-12T00:52:45,639 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33953 2024-11-12T00:52:45,654 DEBUG [M:0;6928b69dca1b:45925 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6928b69dca1b:45925 2024-11-12T00:52:45,655 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6928b69dca1b,45925,1731372764943 2024-11-12T00:52:45,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T00:52:45,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T00:52:45,663 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6928b69dca1b,45925,1731372764943 2024-11-12T00:52:45,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T00:52:45,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T00:52:45,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T00:52:45,685 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T00:52:45,686 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6928b69dca1b,45925,1731372764943 from backup master directory 2024-11-12T00:52:45,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6928b69dca1b,45925,1731372764943 2024-11-12T00:52:45,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T00:52:45,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T00:52:45,689 WARN [master/6928b69dca1b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T00:52:45,690 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6928b69dca1b,45925,1731372764943 2024-11-12T00:52:45,692 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T00:52:45,694 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T00:52:45,748 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/hbase.id] with ID: fdc4e098-f842-461c-a9b9-51c8c13368ac 2024-11-12T00:52:45,748 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/.tmp/hbase.id 2024-11-12T00:52:45,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741826_1002 (size=42) 2024-11-12T00:52:46,163 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/.tmp/hbase.id]:[hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/hbase.id] 2024-11-12T00:52:46,214 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T00:52:46,218 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T00:52:46,236 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-12T00:52:46,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T00:52:46,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T00:52:46,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741827_1003 (size=196) 2024-11-12T00:52:46,673 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T00:52:46,676 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T00:52:46,694 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:46,698 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T00:52:46,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741828_1004 (size=1189) 2024-11-12T00:52:47,153 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store 2024-11-12T00:52:47,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741829_1005 (size=34) 2024-11-12T00:52:47,190 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T00:52:47,195 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:52:47,197 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T00:52:47,197 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T00:52:47,197 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T00:52:47,200 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T00:52:47,200 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T00:52:47,200 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T00:52:47,202 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731372767196Disabling compacts and flushes for region at 1731372767196Disabling writes for close at 1731372767200 (+4 ms)Writing region close event to WAL at 1731372767200Closed at 1731372767200 2024-11-12T00:52:47,205 WARN [master/6928b69dca1b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/.initializing 2024-11-12T00:52:47,206 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/WALs/6928b69dca1b,45925,1731372764943 2024-11-12T00:52:47,217 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T00:52:47,238 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6928b69dca1b%2C45925%2C1731372764943, suffix=, logDir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/WALs/6928b69dca1b,45925,1731372764943, archiveDir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/oldWALs, maxLogs=10 2024-11-12T00:52:47,269 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/WALs/6928b69dca1b,45925,1731372764943/6928b69dca1b%2C45925%2C1731372764943.1731372767244, exclude list is [], retry=0 2024-11-12T00:52:47,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40185,DS-04b1c5dc-6d16-42ad-9f87-e9415d4ad469,DISK] 2024-11-12T00:52:47,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T00:52:47,343 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/WALs/6928b69dca1b,45925,1731372764943/6928b69dca1b%2C45925%2C1731372764943.1731372767244 2024-11-12T00:52:47,345 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39725:39725)] 2024-11-12T00:52:47,346 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T00:52:47,346 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:52:47,350 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T00:52:47,351 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T00:52:47,395 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T00:52:47,424 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T00:52:47,429 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:47,432 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T00:52:47,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T00:52:47,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T00:52:47,439 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:47,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:52:47,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T00:52:47,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T00:52:47,444 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:47,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:52:47,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T00:52:47,449 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T00:52:47,449 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:47,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:52:47,451 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T00:52:47,456 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T00:52:47,458 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T00:52:47,463 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T00:52:47,464 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T00:52:47,469 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T00:52:47,474 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T00:52:47,481 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T00:52:47,483 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68215090, jitterRate=0.016484051942825317}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T00:52:47,492 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731372767364Initializing all the Stores at 1731372767367 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731372767368 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372767368Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372767369 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372767369Cleaning up temporary data from old regions at 1731372767464 (+95 ms)Region opened successfully at 1731372767492 (+28 ms) 2024-11-12T00:52:47,494 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T00:52:47,541 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fb93017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6928b69dca1b/172.17.0.2:0 2024-11-12T00:52:47,585 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T00:52:47,597 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T00:52:47,597 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T00:52:47,600 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T00:52:47,602 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T00:52:47,608 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T00:52:47,608 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T00:52:47,644 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T00:52:47,656 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T00:52:47,659 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T00:52:47,662 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T00:52:47,664 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T00:52:47,666 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T00:52:47,669 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T00:52:47,673 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T00:52:47,675 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T00:52:47,676 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T00:52:47,678 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T00:52:47,699 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T00:52:47,701 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T00:52:47,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T00:52:47,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T00:52:47,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T00:52:47,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T00:52:47,710 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6928b69dca1b,45925,1731372764943, sessionid=0x100316b283a0000, setting cluster-up flag (Was=false) 2024-11-12T00:52:47,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T00:52:47,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T00:52:47,732 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T00:52:47,735 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6928b69dca1b,45925,1731372764943 2024-11-12T00:52:47,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T00:52:47,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T00:52:47,747 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T00:52:47,749 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6928b69dca1b,45925,1731372764943 2024-11-12T00:52:47,757 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T00:52:47,848 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(746): ClusterId : fdc4e098-f842-461c-a9b9-51c8c13368ac 2024-11-12T00:52:47,851 DEBUG [RS:0;6928b69dca1b:33953 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T00:52:47,858 DEBUG [RS:0;6928b69dca1b:33953 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T00:52:47,858 DEBUG [RS:0;6928b69dca1b:33953 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T00:52:47,860 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T00:52:47,863 DEBUG [RS:0;6928b69dca1b:33953 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T00:52:47,864 DEBUG [RS:0;6928b69dca1b:33953 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eb40803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6928b69dca1b/172.17.0.2:0 2024-11-12T00:52:47,874 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T00:52:47,885 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6928b69dca1b:33953 2024-11-12T00:52:47,886 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T00:52:47,889 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T00:52:47,890 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T00:52:47,890 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T00:52:47,896 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(2659): reportForDuty to master=6928b69dca1b,45925,1731372764943 with port=33953, startcode=1731372765559 2024-11-12T00:52:47,895 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6928b69dca1b,45925,1731372764943 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T00:52:47,904 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6928b69dca1b:0, corePoolSize=5, maxPoolSize=5 2024-11-12T00:52:47,904 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6928b69dca1b:0, corePoolSize=5, maxPoolSize=5 2024-11-12T00:52:47,904 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6928b69dca1b:0, corePoolSize=5, maxPoolSize=5 2024-11-12T00:52:47,905 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6928b69dca1b:0, corePoolSize=5, maxPoolSize=5 2024-11-12T00:52:47,905 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6928b69dca1b:0, corePoolSize=10, maxPoolSize=10 2024-11-12T00:52:47,905 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6928b69dca1b:0, corePoolSize=1, maxPoolSize=1 2024-11-12T00:52:47,906 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6928b69dca1b:0, corePoolSize=2, maxPoolSize=2 2024-11-12T00:52:47,906 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6928b69dca1b:0, corePoolSize=1, maxPoolSize=1 2024-11-12T00:52:47,914 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731372797914 2024-11-12T00:52:47,916 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T00:52:47,918 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T00:52:47,921 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T00:52:47,922 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T00:52:47,922 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T00:52:47,923 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T00:52:47,923 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T00:52:47,924 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T00:52:47,927 DEBUG [RS:0;6928b69dca1b:33953 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T00:52:47,934 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:47,940 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:47,941 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T00:52:47,942 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T00:52:47,941 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T00:52:47,943 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T00:52:47,965 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T00:52:47,966 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T00:52:47,968 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6928b69dca1b:0:becomeActiveMaster-HFileCleaner.large.0-1731372767967,5,FailOnTimeoutGroup] 2024-11-12T00:52:47,977 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6928b69dca1b:0:becomeActiveMaster-HFileCleaner.small.0-1731372767969,5,FailOnTimeoutGroup] 2024-11-12T00:52:47,977 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:47,978 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T00:52:47,979 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:47,981 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:47,998 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33381, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T00:52:48,007 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45925 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6928b69dca1b,33953,1731372765559 2024-11-12T00:52:48,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741831_1007 (size=1321) 2024-11-12T00:52:48,010 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45925 {}] master.ServerManager(517): Registering regionserver=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:48,013 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T00:52:48,014 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6 2024-11-12T00:52:48,035 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6 2024-11-12T00:52:48,035 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33743 2024-11-12T00:52:48,035 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T00:52:48,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T00:52:48,042 DEBUG [RS:0;6928b69dca1b:33953 {}] zookeeper.ZKUtil(111): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6928b69dca1b,33953,1731372765559 2024-11-12T00:52:48,043 WARN [RS:0;6928b69dca1b:33953 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T00:52:48,043 INFO [RS:0;6928b69dca1b:33953 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T00:52:48,044 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/WALs/6928b69dca1b,33953,1731372765559 2024-11-12T00:52:48,045 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6928b69dca1b,33953,1731372765559] 2024-11-12T00:52:48,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741832_1008 (size=32) 2024-11-12T00:52:48,054 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:52:48,058 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T00:52:48,064 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T00:52:48,065 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:48,066 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T00:52:48,067 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T00:52:48,070 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T00:52:48,070 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:48,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T00:52:48,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T00:52:48,077 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T00:52:48,077 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:48,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T00:52:48,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T00:52:48,088 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T00:52:48,089 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:48,091 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T00:52:48,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T00:52:48,093 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T00:52:48,094 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740 2024-11-12T00:52:48,095 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740 2024-11-12T00:52:48,099 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T00:52:48,100 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T00:52:48,101 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T00:52:48,105 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T00:52:48,112 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T00:52:48,113 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63971355, jitterRate=-0.046752527356147766}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T00:52:48,116 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T00:52:48,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731372768054Initializing all the Stores at 1731372768057 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731372768057Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731372768057Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372768058 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731372768058Cleaning up temporary data from old regions at 1731372768100 (+42 ms)Region opened successfully at 1731372768120 (+20 ms) 2024-11-12T00:52:48,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T00:52:48,120 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T00:52:48,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T00:52:48,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T00:52:48,121 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T00:52:48,122 INFO [RS:0;6928b69dca1b:33953 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T00:52:48,123 INFO [RS:0;6928b69dca1b:33953 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:48,126 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T00:52:48,126 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T00:52:48,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731372768120Disabling compacts and flushes for region at 1731372768120Disabling writes for close at 1731372768120Writing region close event to WAL at 1731372768125 (+5 ms)Closed at 1731372768126 (+1 ms) 2024-11-12T00:52:48,131 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T00:52:48,131 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T00:52:48,134 INFO [RS:0;6928b69dca1b:33953 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T00:52:48,135 INFO [RS:0;6928b69dca1b:33953 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:48,136 DEBUG [RS:0;6928b69dca1b:33953 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6928b69dca1b:0, corePoolSize=1, maxPoolSize=1 2024-11-12T00:52:48,136 DEBUG [RS:0;6928b69dca1b:33953 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6928b69dca1b:0, corePoolSize=1, maxPoolSize=1 2024-11-12T00:52:48,137 DEBUG [RS:0;6928b69dca1b:33953 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6928b69dca1b:0, corePoolSize=1, maxPoolSize=1 2024-11-12T00:52:48,137 DEBUG [RS:0;6928b69dca1b:33953 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6928b69dca1b:0, corePoolSize=1, maxPoolSize=1 2024-11-12T00:52:48,137 DEBUG [RS:0;6928b69dca1b:33953 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6928b69dca1b:0, corePoolSize=1, maxPoolSize=1 2024-11-12T00:52:48,137 DEBUG [RS:0;6928b69dca1b:33953 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6928b69dca1b:0, corePoolSize=2, maxPoolSize=2 2024-11-12T00:52:48,137 DEBUG [RS:0;6928b69dca1b:33953 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0, corePoolSize=1, maxPoolSize=1 2024-11-12T00:52:48,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T00:52:48,138 DEBUG [RS:0;6928b69dca1b:33953 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6928b69dca1b:0, corePoolSize=1, maxPoolSize=1 2024-11-12T00:52:48,138 DEBUG [RS:0;6928b69dca1b:33953 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6928b69dca1b:0, corePoolSize=1, maxPoolSize=1 2024-11-12T00:52:48,138 DEBUG [RS:0;6928b69dca1b:33953 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6928b69dca1b:0, corePoolSize=1, maxPoolSize=1 2024-11-12T00:52:48,138 DEBUG [RS:0;6928b69dca1b:33953 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6928b69dca1b:0, corePoolSize=1, maxPoolSize=1 2024-11-12T00:52:48,138 DEBUG [RS:0;6928b69dca1b:33953 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6928b69dca1b:0, corePoolSize=1, maxPoolSize=1 2024-11-12T00:52:48,138 DEBUG [RS:0;6928b69dca1b:33953 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6928b69dca1b:0, corePoolSize=3, maxPoolSize=3 2024-11-12T00:52:48,139 DEBUG [RS:0;6928b69dca1b:33953 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0, corePoolSize=3, maxPoolSize=3 2024-11-12T00:52:48,143 INFO [RS:0;6928b69dca1b:33953 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:48,143 INFO [RS:0;6928b69dca1b:33953 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:48,143 INFO [RS:0;6928b69dca1b:33953 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:48,143 INFO [RS:0;6928b69dca1b:33953 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:48,144 INFO [RS:0;6928b69dca1b:33953 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:48,144 INFO [RS:0;6928b69dca1b:33953 {}] hbase.ChoreService(168): Chore ScheduledChore name=6928b69dca1b,33953,1731372765559-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T00:52:48,149 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T00:52:48,153 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T00:52:48,170 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T00:52:48,172 INFO [RS:0;6928b69dca1b:33953 {}] hbase.ChoreService(168): Chore ScheduledChore name=6928b69dca1b,33953,1731372765559-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:48,173 INFO [RS:0;6928b69dca1b:33953 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:48,173 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.Replication(171): 6928b69dca1b,33953,1731372765559 started 2024-11-12T00:52:48,196 INFO [RS:0;6928b69dca1b:33953 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:48,197 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1482): Serving as 6928b69dca1b,33953,1731372765559, RpcServer on 6928b69dca1b/172.17.0.2:33953, sessionid=0x100316b283a0001 2024-11-12T00:52:48,198 DEBUG [RS:0;6928b69dca1b:33953 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T00:52:48,198 DEBUG [RS:0;6928b69dca1b:33953 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6928b69dca1b,33953,1731372765559 2024-11-12T00:52:48,198 DEBUG [RS:0;6928b69dca1b:33953 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6928b69dca1b,33953,1731372765559' 2024-11-12T00:52:48,198 DEBUG [RS:0;6928b69dca1b:33953 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T00:52:48,200 DEBUG [RS:0;6928b69dca1b:33953 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T00:52:48,201 DEBUG [RS:0;6928b69dca1b:33953 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T00:52:48,202 DEBUG [RS:0;6928b69dca1b:33953 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T00:52:48,202 DEBUG [RS:0;6928b69dca1b:33953 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6928b69dca1b,33953,1731372765559 2024-11-12T00:52:48,202 DEBUG [RS:0;6928b69dca1b:33953 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6928b69dca1b,33953,1731372765559' 2024-11-12T00:52:48,202 DEBUG [RS:0;6928b69dca1b:33953 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T00:52:48,203 DEBUG [RS:0;6928b69dca1b:33953 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T00:52:48,204 DEBUG [RS:0;6928b69dca1b:33953 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T00:52:48,204 INFO [RS:0;6928b69dca1b:33953 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T00:52:48,204 INFO [RS:0;6928b69dca1b:33953 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T00:52:48,304 WARN [6928b69dca1b:45925 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T00:52:48,309 INFO [RS:0;6928b69dca1b:33953 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T00:52:48,313 INFO [RS:0;6928b69dca1b:33953 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6928b69dca1b%2C33953%2C1731372765559, suffix=, logDir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/WALs/6928b69dca1b,33953,1731372765559, archiveDir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/oldWALs, maxLogs=32 2024-11-12T00:52:48,333 DEBUG [RS:0;6928b69dca1b:33953 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/WALs/6928b69dca1b,33953,1731372765559/6928b69dca1b%2C33953%2C1731372765559.1731372768315, exclude list is [], retry=0 2024-11-12T00:52:48,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40185,DS-04b1c5dc-6d16-42ad-9f87-e9415d4ad469,DISK] 2024-11-12T00:52:48,346 INFO [RS:0;6928b69dca1b:33953 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/WALs/6928b69dca1b,33953,1731372765559/6928b69dca1b%2C33953%2C1731372765559.1731372768315 2024-11-12T00:52:48,347 DEBUG [RS:0;6928b69dca1b:33953 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39725:39725)] 2024-11-12T00:52:48,558 DEBUG [6928b69dca1b:45925 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-12T00:52:48,571 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:48,578 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6928b69dca1b,33953,1731372765559, state=OPENING 2024-11-12T00:52:48,583 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T00:52:48,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T00:52:48,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T00:52:48,586 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T00:52:48,586 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T00:52:48,588 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T00:52:48,590 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:52:48,766 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T00:52:48,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42345, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T00:52:48,788 INFO [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T00:52:48,788 INFO [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T00:52:48,789 INFO [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T00:52:48,793 INFO [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6928b69dca1b%2C33953%2C1731372765559.meta, suffix=.meta, logDir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/WALs/6928b69dca1b,33953,1731372765559, archiveDir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/oldWALs, maxLogs=32 2024-11-12T00:52:48,817 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/WALs/6928b69dca1b,33953,1731372765559/6928b69dca1b%2C33953%2C1731372765559.meta.1731372768796.meta, exclude list is [], retry=0 2024-11-12T00:52:48,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40185,DS-04b1c5dc-6d16-42ad-9f87-e9415d4ad469,DISK] 2024-11-12T00:52:48,827 INFO [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/WALs/6928b69dca1b,33953,1731372765559/6928b69dca1b%2C33953%2C1731372765559.meta.1731372768796.meta 2024-11-12T00:52:48,828 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39725:39725)] 2024-11-12T00:52:48,828 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T00:52:48,831 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T00:52:48,834 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T00:52:48,841 INFO [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T00:52:48,847 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T00:52:48,848 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:52:48,848 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T00:52:48,849 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T00:52:48,853 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T00:52:48,855 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T00:52:48,855 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:48,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T00:52:48,857 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T00:52:48,859 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T00:52:48,859 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:48,860 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T00:52:48,860 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T00:52:48,862 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T00:52:48,862 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:48,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T00:52:48,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T00:52:48,865 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T00:52:48,865 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:48,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T00:52:48,867 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T00:52:48,868 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740 2024-11-12T00:52:48,872 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740 2024-11-12T00:52:48,875 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T00:52:48,875 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T00:52:48,876 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T00:52:48,880 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T00:52:48,882 INFO [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63851885, jitterRate=-0.048532769083976746}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T00:52:48,882 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T00:52:48,884 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731372768849Writing region info on filesystem at 1731372768850 (+1 ms)Initializing all the Stores at 1731372768852 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731372768852Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731372768853 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372768853Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731372768853Cleaning up temporary data from old regions at 1731372768875 (+22 ms)Running coprocessor post-open hooks at 1731372768882 (+7 ms)Region opened successfully at 1731372768884 (+2 ms) 2024-11-12T00:52:48,894 INFO [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731372768757 2024-11-12T00:52:48,912 DEBUG [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T00:52:48,913 INFO [RS_OPEN_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T00:52:48,916 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:48,920 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6928b69dca1b,33953,1731372765559, state=OPEN 2024-11-12T00:52:48,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T00:52:48,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T00:52:48,923 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T00:52:48,924 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T00:52:48,924 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:48,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T00:52:48,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6928b69dca1b,33953,1731372765559 in 334 msec 2024-11-12T00:52:48,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T00:52:48,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 795 msec 2024-11-12T00:52:48,940 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T00:52:48,940 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T00:52:48,960 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:52:48,961 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:52:48,981 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:52:48,984 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51139, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:52:49,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2040 sec 2024-11-12T00:52:49,007 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731372769007, completionTime=-1 2024-11-12T00:52:49,010 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-12T00:52:49,010 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T00:52:49,039 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-12T00:52:49,039 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731372829039 2024-11-12T00:52:49,039 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731372889039 2024-11-12T00:52:49,039 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-12T00:52:49,041 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6928b69dca1b,45925,1731372764943-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:49,042 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6928b69dca1b,45925,1731372764943-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:49,042 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6928b69dca1b,45925,1731372764943-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:49,043 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6928b69dca1b:45925, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:49,044 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:49,045 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:49,051 DEBUG [master/6928b69dca1b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T00:52:49,079 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.388sec 2024-11-12T00:52:49,080 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T00:52:49,082 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T00:52:49,084 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T00:52:49,084 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T00:52:49,085 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T00:52:49,086 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6928b69dca1b,45925,1731372764943-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T00:52:49,086 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6928b69dca1b,45925,1731372764943-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T00:52:49,097 DEBUG [master/6928b69dca1b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T00:52:49,098 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T00:52:49,098 INFO [master/6928b69dca1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6928b69dca1b,45925,1731372764943-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T00:52:49,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35dab4e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:49,158 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T00:52:49,159 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T00:52:49,162 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:52:49,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:52:49,174 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:52:49,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:52:49,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:52:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f835d7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:52:49,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:52:49,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:52:49,187 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:52:49,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a816d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:49,191 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:52:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:52:49,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:52:49,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:52:49,234 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6928b69dca1b,45925,1731372764943 2024-11-12T00:52:49,247 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=479, ProcessCount=11, AvailableMemoryMB=6292 2024-11-12T00:52:49,259 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T00:52:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 6928b69dca1b,45925,1731372764943 2024-11-12T00:52:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@11630d03 2024-11-12T00:52:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T00:52:49,277 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T00:52:49,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T00:52:49,285 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T00:52:49,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T00:52:49,300 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T00:52:49,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-12T00:52:49,303 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:49,306 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T00:52:49,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T00:52:49,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741835_1011 (size=1061) 2024-11-12T00:52:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T00:52:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T00:52:49,728 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6 2024-11-12T00:52:49,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741836_1012 (size=53) 2024-11-12T00:52:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T00:52:50,140 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:52:50,140 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8f6f5fa6d51a99bb28036e719e157e24, disabling compactions & flushes 2024-11-12T00:52:50,140 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:50,140 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:50,140 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. after waiting 0 ms 2024-11-12T00:52:50,140 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:50,141 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:50,141 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8f6f5fa6d51a99bb28036e719e157e24: Waiting for close lock at 1731372770140Disabling compacts and flushes for region at 1731372770140Disabling writes for close at 1731372770140Writing region close event to WAL at 1731372770141 (+1 ms)Closed at 1731372770141 2024-11-12T00:52:50,146 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T00:52:50,152 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731372770146"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731372770146"}]},"ts":"1731372770146"} 2024-11-12T00:52:50,159 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T00:52:50,162 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T00:52:50,166 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372770162"}]},"ts":"1731372770162"} 2024-11-12T00:52:50,172 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T00:52:50,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f6f5fa6d51a99bb28036e719e157e24, ASSIGN}] 2024-11-12T00:52:50,177 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f6f5fa6d51a99bb28036e719e157e24, ASSIGN 2024-11-12T00:52:50,179 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f6f5fa6d51a99bb28036e719e157e24, ASSIGN; state=OFFLINE, location=6928b69dca1b,33953,1731372765559; forceNewPlan=false, retain=false 2024-11-12T00:52:50,331 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8f6f5fa6d51a99bb28036e719e157e24, regionState=OPENING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:50,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f6f5fa6d51a99bb28036e719e157e24, ASSIGN because future has completed 2024-11-12T00:52:50,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:52:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T00:52:50,505 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:50,506 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} 2024-11-12T00:52:50,506 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:50,506 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:52:50,507 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:50,507 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:50,510 INFO [StoreOpener-8f6f5fa6d51a99bb28036e719e157e24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:50,515 INFO [StoreOpener-8f6f5fa6d51a99bb28036e719e157e24-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:52:50,515 INFO [StoreOpener-8f6f5fa6d51a99bb28036e719e157e24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f6f5fa6d51a99bb28036e719e157e24 columnFamilyName A 2024-11-12T00:52:50,515 DEBUG [StoreOpener-8f6f5fa6d51a99bb28036e719e157e24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:50,517 INFO [StoreOpener-8f6f5fa6d51a99bb28036e719e157e24-1 {}] regionserver.HStore(327): Store=8f6f5fa6d51a99bb28036e719e157e24/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:52:50,518 INFO [StoreOpener-8f6f5fa6d51a99bb28036e719e157e24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:50,526 INFO [StoreOpener-8f6f5fa6d51a99bb28036e719e157e24-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:52:50,527 INFO [StoreOpener-8f6f5fa6d51a99bb28036e719e157e24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f6f5fa6d51a99bb28036e719e157e24 columnFamilyName B 2024-11-12T00:52:50,527 DEBUG [StoreOpener-8f6f5fa6d51a99bb28036e719e157e24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:50,529 INFO [StoreOpener-8f6f5fa6d51a99bb28036e719e157e24-1 {}] regionserver.HStore(327): Store=8f6f5fa6d51a99bb28036e719e157e24/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:52:50,529 INFO [StoreOpener-8f6f5fa6d51a99bb28036e719e157e24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:50,532 INFO [StoreOpener-8f6f5fa6d51a99bb28036e719e157e24-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:52:50,533 INFO [StoreOpener-8f6f5fa6d51a99bb28036e719e157e24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f6f5fa6d51a99bb28036e719e157e24 columnFamilyName C 2024-11-12T00:52:50,533 DEBUG [StoreOpener-8f6f5fa6d51a99bb28036e719e157e24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:52:50,535 INFO [StoreOpener-8f6f5fa6d51a99bb28036e719e157e24-1 {}] regionserver.HStore(327): Store=8f6f5fa6d51a99bb28036e719e157e24/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:52:50,536 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:50,536 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:50,539 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:50,540 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:50,544 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:50,544 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:50,545 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T00:52:50,557 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:50,569 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T00:52:50,570 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8f6f5fa6d51a99bb28036e719e157e24; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65176360, jitterRate=-0.02879655361175537}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T00:52:50,571 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:50,573 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8f6f5fa6d51a99bb28036e719e157e24: Running coprocessor pre-open hook at 1731372770507Writing region info on filesystem at 1731372770507Initializing all the Stores at 1731372770510 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372770510Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372770510Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372770510Cleaning up temporary data from old regions at 1731372770544 (+34 ms)Running coprocessor post-open hooks at 1731372770571 (+27 ms)Region opened successfully at 1731372770573 (+2 ms) 2024-11-12T00:52:50,576 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., pid=6, masterSystemTime=1731372770495 2024-11-12T00:52:50,584 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:50,584 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:50,584 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8f6f5fa6d51a99bb28036e719e157e24, regionState=OPEN, openSeqNum=2, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:50,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:52:50,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T00:52:50,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 in 257 msec 2024-11-12T00:52:50,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T00:52:50,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f6f5fa6d51a99bb28036e719e157e24, ASSIGN in 430 msec 2024-11-12T00:52:50,617 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T00:52:50,617 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372770617"}]},"ts":"1731372770617"} 2024-11-12T00:52:50,621 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T00:52:50,624 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T00:52:50,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3350 sec 2024-11-12T00:52:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T00:52:51,468 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:52:51,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acda54d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,475 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:52:51,475 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:52:51,477 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:52:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:52:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:52:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328e9320, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:52:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:52:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:52:51,481 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:52:51,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ec4a323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,484 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:52:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:52:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:52:51,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:52:51,493 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T00:52:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 6928b69dca1b,45925,1731372764943 2024-11-12T00:52:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@30e64719 2024-11-12T00:52:51,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T00:52:51,500 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T00:52:51,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4acbd0d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:52:51,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:52:51,507 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:52:51,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:52:51,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:52:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6b0c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:52:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:52:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:52:51,512 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:52:51,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740f65e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd90ee6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:52:51,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:52:51,518 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:52:51,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:52:51,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:52:51,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a08f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:52:51,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:52:51,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:52:51,523 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:52:51,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577d2795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:52:51,528 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:52:51,530 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:52:51,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:52:51,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:52:51,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79382fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:52:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:52:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:52:51,534 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:52:51,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c55d99d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bba02c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,537 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:52:51,537 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:52:51,546 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:52:51,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:52:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:52:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6ed1ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:52:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:52:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:52:51,552 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:52:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62dac934, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@113d617f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:52:51,557 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:52:51,558 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:52:51,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:52:51,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:52:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c6d5516, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:52:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:52:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:52:51,562 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:52:51,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596e3a6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:52:51,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:52:51,572 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:52:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:52:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:52:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e9d3a67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:52:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:52:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:52:51,576 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:52:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b3d6975, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3ddbdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:52:51,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:52:51,584 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:52:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:52:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:52:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cc5ceca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:52:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:52:51,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:52:51,588 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:52:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46783332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:52:51,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:52:51,596 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:52:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:52:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:52:51,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e877dc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:52:51,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:52:51,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:52:51,601 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:52:51,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35351cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3111333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,611 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:52:51,611 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:52:51,613 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:52:51,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:52:51,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:52:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c39882a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:52:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:52:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:52:51,618 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:52:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21616163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:52:51,624 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:52:51,625 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:52:51,625 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:52:51,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:52:51,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:52:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:52:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:52:51,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:52:51,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:52:51,629 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:52:51,630 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:52:51,630 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:52:51,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:52:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:52:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:52:51,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:52:51,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:52:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:52:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:52:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:52:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-12T00:52:51,645 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:52:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:52:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:52:51,648 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:52:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:52:51,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:52:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T00:52:51,655 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:52:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:52:51,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:52:51,693 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:52:51,694 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:52:51,696 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:52:51,698 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:52:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:52:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:52:51,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:52:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:52:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:52:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:52:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:52:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:52:51,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:52:51,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:52:51,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:52:51,717 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:52:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:52:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:52:51,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:52:51,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:52:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T00:52:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:51,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T00:52:51,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:52:51,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:51,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:52:51,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:51,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:52:51,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:51,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T00:52:51,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:51,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:51,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:51,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:51,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:51,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372831955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T00:52:51,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/82e45c82574c44b6b5ca96ec3fff8fcf is 50, key is test_row_0/A:col10/1731372771787/Put/seqid=0 2024-11-12T00:52:51,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372831962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:51,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372831968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372831969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372831969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741837_1013 (size=12001) 2024-11-12T00:52:52,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/82e45c82574c44b6b5ca96ec3fff8fcf 2024-11-12T00:52:52,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T00:52:52,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:52,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:52,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:52,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:52,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372832137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372832138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372832140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372832140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372832140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/710a69c7698a433d890c941434dc6342 is 50, key is test_row_0/B:col10/1731372771787/Put/seqid=0 2024-11-12T00:52:52,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741838_1014 (size=12001) 2024-11-12T00:52:52,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/710a69c7698a433d890c941434dc6342 2024-11-12T00:52:52,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T00:52:52,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:52,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:52,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:52,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:52,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:52,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/82beca1bdb3c4e28998cf9a32529f9d2 is 50, key is test_row_0/C:col10/1731372771787/Put/seqid=0 2024-11-12T00:52:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T00:52:52,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741839_1015 (size=12001) 2024-11-12T00:52:52,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/82beca1bdb3c4e28998cf9a32529f9d2 2024-11-12T00:52:52,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/82e45c82574c44b6b5ca96ec3fff8fcf as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/82e45c82574c44b6b5ca96ec3fff8fcf 2024-11-12T00:52:52,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/82e45c82574c44b6b5ca96ec3fff8fcf, entries=150, sequenceid=13, filesize=11.7 K 2024-11-12T00:52:52,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/710a69c7698a433d890c941434dc6342 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/710a69c7698a433d890c941434dc6342 2024-11-12T00:52:52,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372832356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372832357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372832357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372832358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372832371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/710a69c7698a433d890c941434dc6342, entries=150, sequenceid=13, filesize=11.7 K 2024-11-12T00:52:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/82beca1bdb3c4e28998cf9a32529f9d2 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/82beca1bdb3c4e28998cf9a32529f9d2 2024-11-12T00:52:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T00:52:52,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:52,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:52,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:52,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:52,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:52,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/82beca1bdb3c4e28998cf9a32529f9d2, entries=150, sequenceid=13, filesize=11.7 K 2024-11-12T00:52:52,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8f6f5fa6d51a99bb28036e719e157e24 in 668ms, sequenceid=13, compaction requested=false 2024-11-12T00:52:52,462 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T00:52:52,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:52,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T00:52:52,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:52,582 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T00:52:52,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:52:52,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:52,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:52:52,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:52,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:52:52,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:52,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/922d8e82128d42c0a9e4e5f8600a4f63 is 50, key is test_row_0/A:col10/1731372771958/Put/seqid=0 2024-11-12T00:52:52,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741840_1016 (size=12001) 2024-11-12T00:52:52,673 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/922d8e82128d42c0a9e4e5f8600a4f63 2024-11-12T00:52:52,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/9c0f15d895454b8a91565d56266f1629 is 50, key is test_row_0/B:col10/1731372771958/Put/seqid=0 2024-11-12T00:52:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:52,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:52,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741841_1017 (size=12001) 2024-11-12T00:52:52,758 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/9c0f15d895454b8a91565d56266f1629 2024-11-12T00:52:52,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372832762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372832771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372832772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T00:52:52,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/7cf39364d2fb4299a603109a18b10ff9 is 50, key is test_row_0/C:col10/1731372771958/Put/seqid=0 2024-11-12T00:52:52,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372832752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372832780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741842_1018 (size=12001) 2024-11-12T00:52:52,822 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/7cf39364d2fb4299a603109a18b10ff9 2024-11-12T00:52:52,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/922d8e82128d42c0a9e4e5f8600a4f63 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/922d8e82128d42c0a9e4e5f8600a4f63 2024-11-12T00:52:52,878 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/922d8e82128d42c0a9e4e5f8600a4f63, entries=150, sequenceid=33, filesize=11.7 K 2024-11-12T00:52:52,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/9c0f15d895454b8a91565d56266f1629 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/9c0f15d895454b8a91565d56266f1629 2024-11-12T00:52:52,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372832898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,904 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/9c0f15d895454b8a91565d56266f1629, entries=150, sequenceid=33, filesize=11.7 K 2024-11-12T00:52:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372832898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372832898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/7cf39364d2fb4299a603109a18b10ff9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/7cf39364d2fb4299a603109a18b10ff9 2024-11-12T00:52:52,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372832918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372832918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:52,940 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/7cf39364d2fb4299a603109a18b10ff9, entries=150, sequenceid=33, filesize=11.7 K 2024-11-12T00:52:52,944 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8f6f5fa6d51a99bb28036e719e157e24 in 361ms, sequenceid=33, compaction requested=false 2024-11-12T00:52:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-12T00:52:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-12T00:52:52,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T00:52:52,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2910 sec 2024-11-12T00:52:52,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.3180 sec 2024-11-12T00:52:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:53,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T00:52:53,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:52:53,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:53,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:52:53,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:53,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:52:53,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:53,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/a3bd343a5d2949a8b3c7fbdb2690e3b9 is 50, key is test_row_0/A:col10/1731372772771/Put/seqid=0 2024-11-12T00:52:53,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741843_1019 (size=12001) 2024-11-12T00:52:53,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/a3bd343a5d2949a8b3c7fbdb2690e3b9 2024-11-12T00:52:53,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372833184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372833189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372833189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372833200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/770214db633845a7ac41e1ebd2b2af73 is 50, key is test_row_0/B:col10/1731372772771/Put/seqid=0 2024-11-12T00:52:53,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372833205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741844_1020 (size=12001) 2024-11-12T00:52:53,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/770214db633845a7ac41e1ebd2b2af73 2024-11-12T00:52:53,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/590740818d9f4f42a168a8e8c637117c is 50, key is test_row_0/C:col10/1731372772771/Put/seqid=0 2024-11-12T00:52:53,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741845_1021 (size=12001) 2024-11-12T00:52:53,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/590740818d9f4f42a168a8e8c637117c 2024-11-12T00:52:53,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372833319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372833320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372833321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372833323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/a3bd343a5d2949a8b3c7fbdb2690e3b9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/a3bd343a5d2949a8b3c7fbdb2690e3b9 2024-11-12T00:52:53,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372833326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/a3bd343a5d2949a8b3c7fbdb2690e3b9, entries=150, sequenceid=47, filesize=11.7 K 2024-11-12T00:52:53,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/770214db633845a7ac41e1ebd2b2af73 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/770214db633845a7ac41e1ebd2b2af73 2024-11-12T00:52:53,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/770214db633845a7ac41e1ebd2b2af73, entries=150, sequenceid=47, filesize=11.7 K 2024-11-12T00:52:53,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/590740818d9f4f42a168a8e8c637117c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/590740818d9f4f42a168a8e8c637117c 2024-11-12T00:52:53,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/590740818d9f4f42a168a8e8c637117c, entries=150, sequenceid=47, filesize=11.7 K 2024-11-12T00:52:53,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8f6f5fa6d51a99bb28036e719e157e24 in 294ms, sequenceid=47, compaction requested=true 2024-11-12T00:52:53,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:53,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:52:53,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:53,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:52:53,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:53,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:52:53,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:52:53,436 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:52:53,436 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:52:53,440 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:52:53,442 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/B is initiating minor compaction (all files) 2024-11-12T00:52:53,443 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/B in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:53,443 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/710a69c7698a433d890c941434dc6342, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/9c0f15d895454b8a91565d56266f1629, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/770214db633845a7ac41e1ebd2b2af73] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=35.2 K 2024-11-12T00:52:53,444 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:52:53,445 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 710a69c7698a433d890c941434dc6342, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731372771761 2024-11-12T00:52:53,444 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/A is initiating minor compaction (all files) 2024-11-12T00:52:53,446 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/A in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:53,446 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c0f15d895454b8a91565d56266f1629, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731372771952 2024-11-12T00:52:53,447 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 770214db633845a7ac41e1ebd2b2af73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731372772762 2024-11-12T00:52:53,448 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/82e45c82574c44b6b5ca96ec3fff8fcf, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/922d8e82128d42c0a9e4e5f8600a4f63, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/a3bd343a5d2949a8b3c7fbdb2690e3b9] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=35.2 K 2024-11-12T00:52:53,451 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82e45c82574c44b6b5ca96ec3fff8fcf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731372771761 2024-11-12T00:52:53,452 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 922d8e82128d42c0a9e4e5f8600a4f63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731372771952 2024-11-12T00:52:53,454 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3bd343a5d2949a8b3c7fbdb2690e3b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731372772762 2024-11-12T00:52:53,521 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#B#compaction#9 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:52:53,522 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#A#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:52:53,527 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/69876bcf13b04d1b979309fc2d9ba1ea is 50, key is test_row_0/A:col10/1731372772771/Put/seqid=0 2024-11-12T00:52:53,533 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/6bddb507666d4fb5b6987477f08efecb is 50, key is test_row_0/B:col10/1731372772771/Put/seqid=0 2024-11-12T00:52:53,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T00:52:53,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:52:53,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:53,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:52:53,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:53,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:52:53,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:53,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/ed4ca10f71ab4b1ba67ce0f6d1d47871 is 50, key is test_row_0/A:col10/1731372773175/Put/seqid=0 2024-11-12T00:52:53,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741846_1022 (size=12104) 2024-11-12T00:52:53,587 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/69876bcf13b04d1b979309fc2d9ba1ea as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/69876bcf13b04d1b979309fc2d9ba1ea 2024-11-12T00:52:53,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741847_1023 (size=12104) 2024-11-12T00:52:53,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372833576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372833582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372833587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,619 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/6bddb507666d4fb5b6987477f08efecb as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/6bddb507666d4fb5b6987477f08efecb 2024-11-12T00:52:53,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741848_1024 (size=14341) 2024-11-12T00:52:53,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372833595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372833598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,629 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/A of 8f6f5fa6d51a99bb28036e719e157e24 into 69876bcf13b04d1b979309fc2d9ba1ea(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:52:53,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/ed4ca10f71ab4b1ba67ce0f6d1d47871 2024-11-12T00:52:53,629 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:53,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,638 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/A, priority=13, startTime=1731372773418; duration=0sec 2024-11-12T00:52:53,640 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:52:53,640 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:A 2024-11-12T00:52:53,640 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:52:53,643 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/B of 8f6f5fa6d51a99bb28036e719e157e24 into 6bddb507666d4fb5b6987477f08efecb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:52:53,643 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:53,643 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/B, priority=13, startTime=1731372773435; duration=0sec 2024-11-12T00:52:53,644 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:53,644 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:B 2024-11-12T00:52:53,645 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:52:53,645 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/C is initiating minor compaction (all files) 2024-11-12T00:52:53,645 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/C in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:53,646 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/82beca1bdb3c4e28998cf9a32529f9d2, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/7cf39364d2fb4299a603109a18b10ff9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/590740818d9f4f42a168a8e8c637117c] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=35.2 K 2024-11-12T00:52:53,647 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82beca1bdb3c4e28998cf9a32529f9d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731372771761 2024-11-12T00:52:53,648 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cf39364d2fb4299a603109a18b10ff9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731372771952 2024-11-12T00:52:53,649 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 590740818d9f4f42a168a8e8c637117c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731372772762 2024-11-12T00:52:53,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/060c8c762e0544f9b526d744d4210a98 is 50, key is test_row_0/B:col10/1731372773175/Put/seqid=0 2024-11-12T00:52:53,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741849_1025 (size=12001) 2024-11-12T00:52:53,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/060c8c762e0544f9b526d744d4210a98 2024-11-12T00:52:53,683 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#C#compaction#13 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:52:53,684 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/c2ba89d85fee4d52be3497418998d020 is 50, key is test_row_0/C:col10/1731372772771/Put/seqid=0 2024-11-12T00:52:53,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/792347adf7ab466eadd05e62e0f03c69 is 50, key is test_row_0/C:col10/1731372773175/Put/seqid=0 2024-11-12T00:52:53,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372833707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372833707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372833726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372833736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372833736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741850_1026 (size=12001) 2024-11-12T00:52:53,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741851_1027 (size=12104) 2024-11-12T00:52:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T00:52:53,807 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:52:53,808 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:52:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-12T00:52:53,813 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:52:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T00:52:53,815 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:52:53,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:52:53,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372833916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372833917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T00:52:53,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372833946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372833947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372833956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:53,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:53,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:53,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:53,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:53,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:53,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:53,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:54,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:54,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:54,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:54,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T00:52:54,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/792347adf7ab466eadd05e62e0f03c69 2024-11-12T00:52:54,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/ed4ca10f71ab4b1ba67ce0f6d1d47871 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/ed4ca10f71ab4b1ba67ce0f6d1d47871 2024-11-12T00:52:54,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/ed4ca10f71ab4b1ba67ce0f6d1d47871, entries=200, sequenceid=66, filesize=14.0 K 2024-11-12T00:52:54,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/060c8c762e0544f9b526d744d4210a98 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/060c8c762e0544f9b526d744d4210a98 2024-11-12T00:52:54,187 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/c2ba89d85fee4d52be3497418998d020 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/c2ba89d85fee4d52be3497418998d020 2024-11-12T00:52:54,207 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/C of 8f6f5fa6d51a99bb28036e719e157e24 into c2ba89d85fee4d52be3497418998d020(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:52:54,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/060c8c762e0544f9b526d744d4210a98, entries=150, sequenceid=66, filesize=11.7 K 2024-11-12T00:52:54,207 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:54,207 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/C, priority=13, startTime=1731372773435; duration=0sec 2024-11-12T00:52:54,207 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:54,207 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:C 2024-11-12T00:52:54,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/792347adf7ab466eadd05e62e0f03c69 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/792347adf7ab466eadd05e62e0f03c69 2024-11-12T00:52:54,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/792347adf7ab466eadd05e62e0f03c69, entries=150, sequenceid=66, filesize=11.7 K 2024-11-12T00:52:54,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8f6f5fa6d51a99bb28036e719e157e24 in 692ms, sequenceid=66, compaction requested=false 2024-11-12T00:52:54,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:54,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T00:52:54,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:52:54,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:54,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:52:54,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:54,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:52:54,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:54,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/815db4ddd4f24d6bbb6ce59087034acb is 50, key is test_row_0/A:col10/1731372774231/Put/seqid=0 2024-11-12T00:52:54,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:54,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:54,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:54,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:54,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741852_1028 (size=12001) 2024-11-12T00:52:54,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/815db4ddd4f24d6bbb6ce59087034acb 2024-11-12T00:52:54,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372834290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372834297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:54,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372834305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:54,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:54,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372834294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:54,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:54,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372834310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:54,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:54,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/be4bbbc18571437e872ad2970e24262b is 50, key is test_row_0/B:col10/1731372774231/Put/seqid=0 2024-11-12T00:52:54,363 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T00:52:54,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741853_1029 (size=12001) 2024-11-12T00:52:54,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372834426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:54,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372834427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372834427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372834427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372834427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:54,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:54,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:54,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:54,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:54,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T00:52:54,482 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T00:52:54,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:54,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:54,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:54,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:54,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372834636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:54,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372834636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:54,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:54,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372834636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372834637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:54,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:54,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372834637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:54,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:54,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:54,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:54,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:54,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:54,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:54,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/be4bbbc18571437e872ad2970e24262b 2024-11-12T00:52:54,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/9d711b159c28423a97b4666606165356 is 50, key is test_row_0/C:col10/1731372774231/Put/seqid=0 2024-11-12T00:52:54,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741854_1030 (size=12001) 2024-11-12T00:52:54,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/9d711b159c28423a97b4666606165356 2024-11-12T00:52:54,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/815db4ddd4f24d6bbb6ce59087034acb as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/815db4ddd4f24d6bbb6ce59087034acb 2024-11-12T00:52:54,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/815db4ddd4f24d6bbb6ce59087034acb, entries=150, sequenceid=83, filesize=11.7 K 2024-11-12T00:52:54,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/be4bbbc18571437e872ad2970e24262b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/be4bbbc18571437e872ad2970e24262b 2024-11-12T00:52:54,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/be4bbbc18571437e872ad2970e24262b, entries=150, sequenceid=83, filesize=11.7 K 2024-11-12T00:52:54,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/9d711b159c28423a97b4666606165356 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9d711b159c28423a97b4666606165356 2024-11-12T00:52:54,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:54,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:54,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:54,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:54,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:54,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9d711b159c28423a97b4666606165356, entries=150, sequenceid=83, filesize=11.7 K 2024-11-12T00:52:54,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8f6f5fa6d51a99bb28036e719e157e24 in 693ms, sequenceid=83, compaction requested=true 2024-11-12T00:52:54,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:54,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:52:54,926 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:52:54,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:54,927 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:52:54,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:52:54,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:54,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:52:54,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:52:54,928 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:52:54,928 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:52:54,928 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/A is initiating minor compaction (all files) 2024-11-12T00:52:54,928 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/B is initiating minor compaction (all files) 2024-11-12T00:52:54,929 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/A in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:54,929 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/B in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:54,929 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/69876bcf13b04d1b979309fc2d9ba1ea, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/ed4ca10f71ab4b1ba67ce0f6d1d47871, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/815db4ddd4f24d6bbb6ce59087034acb] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=37.5 K 2024-11-12T00:52:54,929 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/6bddb507666d4fb5b6987477f08efecb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/060c8c762e0544f9b526d744d4210a98, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/be4bbbc18571437e872ad2970e24262b] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=35.3 K 2024-11-12T00:52:54,930 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bddb507666d4fb5b6987477f08efecb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731372772762 2024-11-12T00:52:54,930 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69876bcf13b04d1b979309fc2d9ba1ea, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731372772762 2024-11-12T00:52:54,930 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 060c8c762e0544f9b526d744d4210a98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731372773175 2024-11-12T00:52:54,931 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed4ca10f71ab4b1ba67ce0f6d1d47871, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731372773173 2024-11-12T00:52:54,931 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 815db4ddd4f24d6bbb6ce59087034acb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731372773583 2024-11-12T00:52:54,933 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting be4bbbc18571437e872ad2970e24262b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731372773583 2024-11-12T00:52:54,956 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#B#compaction#18 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:52:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T00:52:54,957 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/993e670d99344685b52696ab27b8a958 is 50, key is test_row_0/B:col10/1731372774231/Put/seqid=0 2024-11-12T00:52:54,965 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#A#compaction#19 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:52:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:54,966 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/73592ef238444a6aa034b8725d2b21ba is 50, key is test_row_0/A:col10/1731372774231/Put/seqid=0 2024-11-12T00:52:54,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T00:52:54,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372774293, max timestamp=1731372774972Num uniques 859; 2024-11-12T00:52:54,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:52:54,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372774293, max timestamp=1731372774972Num uniques 859; 2024-11-12T00:52:54,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:52:54,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741855_1031 (size=12207) 2024-11-12T00:52:55,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372774293, max timestamp=1731372774972Num uniques 859; 2024-11-12T00:52:55,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:52:55,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:52:55,011 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/993e670d99344685b52696ab27b8a958 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/993e670d99344685b52696ab27b8a958 2024-11-12T00:52:55,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:52:55,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:52:55,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:52:55,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:52:55,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:52:55,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741856_1032 (size=12207) 2024-11-12T00:52:55,034 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/B of 8f6f5fa6d51a99bb28036e719e157e24 into 993e670d99344685b52696ab27b8a958(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:52:55,035 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:55,035 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/B, priority=13, startTime=1731372774926; duration=0sec 2024-11-12T00:52:55,035 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:52:55,035 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:B 2024-11-12T00:52:55,035 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:52:55,038 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:52:55,038 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/C is initiating minor compaction (all files) 2024-11-12T00:52:55,038 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/C in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:55,039 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/c2ba89d85fee4d52be3497418998d020, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/792347adf7ab466eadd05e62e0f03c69, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9d711b159c28423a97b4666606165356] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=35.3 K 2024-11-12T00:52:55,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/3b8b967e6582488fabe5c3cc91df2a58 is 50, key is test_row_0/A:col10/1731372774293/Put/seqid=0 2024-11-12T00:52:55,043 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/73592ef238444a6aa034b8725d2b21ba as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/73592ef238444a6aa034b8725d2b21ba 2024-11-12T00:52:55,044 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting c2ba89d85fee4d52be3497418998d020, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731372772762 2024-11-12T00:52:55,045 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 792347adf7ab466eadd05e62e0f03c69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731372773175 2024-11-12T00:52:55,047 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d711b159c28423a97b4666606165356, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731372773583 2024-11-12T00:52:55,061 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/A of 8f6f5fa6d51a99bb28036e719e157e24 into 73592ef238444a6aa034b8725d2b21ba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:52:55,061 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:55,062 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/A, priority=13, startTime=1731372774926; duration=0sec 2024-11-12T00:52:55,063 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:55,063 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:A 2024-11-12T00:52:55,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:55,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:55,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:55,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:55,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741857_1033 (size=12001) 2024-11-12T00:52:55,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/3b8b967e6582488fabe5c3cc91df2a58 2024-11-12T00:52:55,092 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#C#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:52:55,093 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/1735cba328b3442b808e4662791d624c is 50, key is test_row_0/C:col10/1731372774231/Put/seqid=0 2024-11-12T00:52:55,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741858_1034 (size=12207) 2024-11-12T00:52:55,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3e17bac0f06a464cb168ef9490a2cf15 is 50, key is test_row_0/B:col10/1731372774293/Put/seqid=0 2024-11-12T00:52:55,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372775094, max timestamp=1731372775112Num uniques 859; 2024-11-12T00:52:55,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:52:55,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372775094, max timestamp=1731372775116Num uniques 859; 2024-11-12T00:52:55,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:52:55,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731372775094, max timestamp=1731372775120Num uniques 859; 2024-11-12T00:52:55,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-12T00:52:55,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741859_1035 (size=12001) 2024-11-12T00:52:55,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:55,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:55,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:55,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:55,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731372775169, max timestamp=1731372775200Num uniques 1018; 2024-11-12T00:52:55,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-12T00:52:55,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731372775169, max timestamp=1731372775200Num uniques 1018; 2024-11-12T00:52:55,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T00:52:55,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731372775169, max timestamp=1731372775200Num uniques 1009; 2024-11-12T00:52:55,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-12T00:52:55,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731372775262, max timestamp=1731372775303Num uniques 1077; 2024-11-12T00:52:55,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731372775262, max timestamp=1731372775303Num uniques 1077; 2024-11-12T00:52:55,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-12T00:52:55,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-12T00:52:55,312 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T00:52:55,312 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T00:52:55,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731372775262, max timestamp=1731372775300Num uniques 1127; 2024-11-12T00:52:55,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-12T00:52:55,315 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T00:52:55,316 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T00:52:55,317 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T00:52:55,317 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T00:52:55,318 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T00:52:55,318 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-12T00:52:55,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731372775338, max timestamp=1731372775360Num uniques 1036; 2024-11-12T00:52:55,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731372775338, max timestamp=1731372775360Num uniques 1036; 2024-11-12T00:52:55,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-12T00:52:55,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-12T00:52:55,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731372775338, max timestamp=1731372775360Num uniques 1036; 2024-11-12T00:52:55,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-12T00:52:55,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:55,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:55,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:55,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:55,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731372775405, max timestamp=1731372775424Num uniques 1095; 2024-11-12T00:52:55,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-12T00:52:55,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731372775405, max timestamp=1731372775424Num uniques 1095; 2024-11-12T00:52:55,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-12T00:52:55,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731372775405, max timestamp=1731372775424Num uniques 1095; 2024-11-12T00:52:55,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-12T00:52:55,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731372775461, max timestamp=1731372775503Num uniques 1054; 2024-11-12T00:52:55,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731372775461, max timestamp=1731372775503Num uniques 1054; 2024-11-12T00:52:55,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731372775461, max timestamp=1731372775503Num uniques 1054; 2024-11-12T00:52:55,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-12T00:52:55,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-12T00:52:55,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-12T00:52:55,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:55,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:55,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:55,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:55,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,543 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/1735cba328b3442b808e4662791d624c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/1735cba328b3442b808e4662791d624c 2024-11-12T00:52:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,570 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/C of 8f6f5fa6d51a99bb28036e719e157e24 into 1735cba328b3442b808e4662791d624c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:52:55,570 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:55,570 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/C, priority=13, startTime=1731372774927; duration=0sec 2024-11-12T00:52:55,571 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:55,571 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:C 2024-11-12T00:52:55,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=363, cellSize=16618, totalHeapSize=37992, min timestamp=1731372775530, max timestamp=1731372775581Num uniques 1113; 2024-11-12T00:52:55,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=363, cellSize=16618, totalHeapSize=37992, min timestamp=1731372775530, max timestamp=1731372775581Num uniques 1113; 2024-11-12T00:52:55,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=363, cellSize=16618, totalHeapSize=37992, min timestamp=1731372775530, max timestamp=1731372775581Num uniques 1113; 2024-11-12T00:52:55,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=16618, suffix heap size=143720,new segment heap size=37992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=363 2024-11-12T00:52:55,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=16618, suffix heap size=143720,new segment heap size=37992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=363 2024-11-12T00:52:55,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=16618, suffix heap size=143720,new segment heap size=37992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=363 2024-11-12T00:52:55,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3e17bac0f06a464cb168ef9490a2cf15 2024-11-12T00:52:55,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/de8029b0b60f440da0cad50b65fd6b21 is 50, key is test_row_0/C:col10/1731372774293/Put/seqid=0 2024-11-12T00:52:55,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372835631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:55,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372835633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:55,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731372775623, max timestamp=1731372775642Num uniques 1222; 2024-11-12T00:52:55,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=55962, new segment data size=12452, suffix heap size=155056,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1222, new segment cells count=272 2024-11-12T00:52:55,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731372775623, max timestamp=1731372775642Num uniques 1222; 2024-11-12T00:52:55,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731372775623, max timestamp=1731372775642Num uniques 1222; 2024-11-12T00:52:55,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=55962, new segment data size=12452, suffix heap size=155056,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1222, new segment cells count=272 2024-11-12T00:52:55,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=55962, new segment data size=12452, suffix heap size=155056,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1222, new segment cells count=272 2024-11-12T00:52:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:55,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741860_1036 (size=12001) 2024-11-12T00:52:55,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/de8029b0b60f440da0cad50b65fd6b21 2024-11-12T00:52:55,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:55,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:55,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:55,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:55,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/3b8b967e6582488fabe5c3cc91df2a58 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3b8b967e6582488fabe5c3cc91df2a58 2024-11-12T00:52:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3b8b967e6582488fabe5c3cc91df2a58, entries=150, sequenceid=104, filesize=11.7 K 2024-11-12T00:52:55,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3e17bac0f06a464cb168ef9490a2cf15 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e17bac0f06a464cb168ef9490a2cf15 2024-11-12T00:52:55,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e17bac0f06a464cb168ef9490a2cf15, entries=150, sequenceid=104, filesize=11.7 K 2024-11-12T00:52:55,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/de8029b0b60f440da0cad50b65fd6b21 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/de8029b0b60f440da0cad50b65fd6b21 2024-11-12T00:52:55,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731372775699, max timestamp=1731372775738Num uniques 1131; 2024-11-12T00:52:55,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731372775699, max timestamp=1731372775738Num uniques 1131; 2024-11-12T00:52:55,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-11-12T00:52:55,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-11-12T00:52:55,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731372775699, max timestamp=1731372775738Num uniques 1131; 2024-11-12T00:52:55,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=6870, suffix heap size=145592,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=150 2024-11-12T00:52:55,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/de8029b0b60f440da0cad50b65fd6b21, entries=150, sequenceid=104, filesize=11.7 K 2024-11-12T00:52:55,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=61.23 KB/62704 for 8f6f5fa6d51a99bb28036e719e157e24 in 792ms, sequenceid=104, compaction requested=false 2024-11-12T00:52:55,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:55,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=81.36 KB heapSize=225.48 KB 2024-11-12T00:52:55,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:52:55,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:52:55,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:52:55,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:52:55,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:52:55,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:52:55,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/14c7e4de5b99451698e5144b451a8a7b is 50, key is test_row_0/A:col10/1731372775771/Put/seqid=0 2024-11-12T00:52:55,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372835812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372835813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372835815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:55,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372835823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:55,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:55,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372835825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:55,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741861_1037 (size=17181) 2024-11-12T00:52:55,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:55,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:55,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:55,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:55,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:55,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372835936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:55,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372835936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:55,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:55,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372835936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:55,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:55,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372835936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372835936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T00:52:56,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:56,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:56,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372836146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372836157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372836157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372836157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372836158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:56,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:56,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/14c7e4de5b99451698e5144b451a8a7b 2024-11-12T00:52:56,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3e5607d3e66c4a54a371add01f840a48 is 50, key is test_row_0/B:col10/1731372775771/Put/seqid=0 2024-11-12T00:52:56,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741862_1038 (size=12301) 2024-11-12T00:52:56,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3e5607d3e66c4a54a371add01f840a48 2024-11-12T00:52:56,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/6da97efcfcbc40e7be4917cb20228b32 is 50, key is test_row_0/C:col10/1731372775771/Put/seqid=0 2024-11-12T00:52:56,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741863_1039 (size=12301) 2024-11-12T00:52:56,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.93 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/6da97efcfcbc40e7be4917cb20228b32 2024-11-12T00:52:56,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/14c7e4de5b99451698e5144b451a8a7b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/14c7e4de5b99451698e5144b451a8a7b 2024-11-12T00:52:56,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/14c7e4de5b99451698e5144b451a8a7b, entries=250, sequenceid=275, filesize=16.8 K 2024-11-12T00:52:56,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3e5607d3e66c4a54a371add01f840a48 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e5607d3e66c4a54a371add01f840a48 2024-11-12T00:52:56,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e5607d3e66c4a54a371add01f840a48, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T00:52:56,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/6da97efcfcbc40e7be4917cb20228b32 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/6da97efcfcbc40e7be4917cb20228b32 2024-11-12T00:52:56,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/6da97efcfcbc40e7be4917cb20228b32, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T00:52:56,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~94.78 KB/97054, heapSize ~265.27 KB/271640, currentSize=87.22 KB/89310 for 8f6f5fa6d51a99bb28036e719e157e24 in 708ms, sequenceid=275, compaction requested=true 2024-11-12T00:52:56,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:56,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:52:56,482 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:52:56,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:56,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:52:56,482 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:52:56,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:56,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:52:56,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:52:56,489 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41389 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:52:56,489 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/A is initiating minor compaction (all files) 2024-11-12T00:52:56,489 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/A in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,489 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/73592ef238444a6aa034b8725d2b21ba, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3b8b967e6582488fabe5c3cc91df2a58, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/14c7e4de5b99451698e5144b451a8a7b] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=40.4 K 2024-11-12T00:52:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:56,490 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:52:56,490 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/B is initiating minor compaction (all files) 2024-11-12T00:52:56,490 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/B in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,490 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/993e670d99344685b52696ab27b8a958, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e17bac0f06a464cb168ef9490a2cf15, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e5607d3e66c4a54a371add01f840a48] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=35.7 K 2024-11-12T00:52:56,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T00:52:56,492 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 993e670d99344685b52696ab27b8a958, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731372773583 2024-11-12T00:52:56,492 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73592ef238444a6aa034b8725d2b21ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731372773583 2024-11-12T00:52:56,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:56,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,493 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e17bac0f06a464cb168ef9490a2cf15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731372774293 2024-11-12T00:52:56,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:56,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,494 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b8b967e6582488fabe5c3cc91df2a58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731372774293 2024-11-12T00:52:56,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,495 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14c7e4de5b99451698e5144b451a8a7b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731372775744 2024-11-12T00:52:56,495 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e5607d3e66c4a54a371add01f840a48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731372775769 2024-11-12T00:52:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:52:56,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:56,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:52:56,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:56,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:52:56,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:56,521 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:52:56,522 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/56735bb1fd804a8f93ca0c97778b5fb1 is 50, key is test_row_0/A:col10/1731372775771/Put/seqid=0 2024-11-12T00:52:56,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/24c538ec2b00491894771b75fc8ff455 is 50, key is test_row_0/A:col10/1731372776490/Put/seqid=0 2024-11-12T00:52:56,544 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#B#compaction#29 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:52:56,545 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/479bc5c78b2542619d5c699418935363 is 50, key is test_row_0/B:col10/1731372775771/Put/seqid=0 2024-11-12T00:52:56,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372836533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372836536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372836548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372836548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372836548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741864_1040 (size=12609) 2024-11-12T00:52:56,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741866_1042 (size=12609) 2024-11-12T00:52:56,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741865_1041 (size=12301) 2024-11-12T00:52:56,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:56,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:56,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372836655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372836666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372836666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372836671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372836668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:56,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:56,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372836876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372836886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372836887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372836887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372836887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:56,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:56,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:56,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:56,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,005 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/56735bb1fd804a8f93ca0c97778b5fb1 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/56735bb1fd804a8f93ca0c97778b5fb1 2024-11-12T00:52:57,019 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/A of 8f6f5fa6d51a99bb28036e719e157e24 into 56735bb1fd804a8f93ca0c97778b5fb1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:52:57,019 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:57,019 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/A, priority=13, startTime=1731372776482; duration=0sec 2024-11-12T00:52:57,020 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:52:57,020 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:A 2024-11-12T00:52:57,020 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:52:57,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/24c538ec2b00491894771b75fc8ff455 2024-11-12T00:52:57,023 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:52:57,026 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/C is initiating minor compaction (all files) 2024-11-12T00:52:57,026 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/C in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:57,027 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/1735cba328b3442b808e4662791d624c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/de8029b0b60f440da0cad50b65fd6b21, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/6da97efcfcbc40e7be4917cb20228b32] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=35.7 K 2024-11-12T00:52:57,030 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/479bc5c78b2542619d5c699418935363 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/479bc5c78b2542619d5c699418935363 2024-11-12T00:52:57,030 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1735cba328b3442b808e4662791d624c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731372773583 2024-11-12T00:52:57,036 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting de8029b0b60f440da0cad50b65fd6b21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731372774293 2024-11-12T00:52:57,040 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6da97efcfcbc40e7be4917cb20228b32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731372775769 2024-11-12T00:52:57,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/1751959ecadf419bbe5640295c8b60cc is 50, key is test_row_0/B:col10/1731372776490/Put/seqid=0 2024-11-12T00:52:57,051 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/B of 8f6f5fa6d51a99bb28036e719e157e24 into 479bc5c78b2542619d5c699418935363(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:52:57,051 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:57,051 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/B, priority=13, startTime=1731372776482; duration=0sec 2024-11-12T00:52:57,051 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:57,051 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:B 2024-11-12T00:52:57,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741867_1043 (size=12301) 2024-11-12T00:52:57,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/1751959ecadf419bbe5640295c8b60cc 2024-11-12T00:52:57,080 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#C#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:52:57,080 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/764e270e3a9847b8806bffdf4bcb27de is 50, key is test_row_0/C:col10/1731372775771/Put/seqid=0 2024-11-12T00:52:57,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/4b9ce6f3b9a5457fa2df5a668518d29f is 50, key is test_row_0/C:col10/1731372776490/Put/seqid=0 2024-11-12T00:52:57,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741869_1045 (size=12301) 2024-11-12T00:52:57,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741868_1044 (size=12609) 2024-11-12T00:52:57,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/4b9ce6f3b9a5457fa2df5a668518d29f 2024-11-12T00:52:57,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:57,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:57,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:57,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:57,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/24c538ec2b00491894771b75fc8ff455 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/24c538ec2b00491894771b75fc8ff455 2024-11-12T00:52:57,142 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/764e270e3a9847b8806bffdf4bcb27de as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/764e270e3a9847b8806bffdf4bcb27de 2024-11-12T00:52:57,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/24c538ec2b00491894771b75fc8ff455, entries=150, sequenceid=295, filesize=12.0 K 2024-11-12T00:52:57,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/1751959ecadf419bbe5640295c8b60cc as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1751959ecadf419bbe5640295c8b60cc 2024-11-12T00:52:57,162 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/C of 8f6f5fa6d51a99bb28036e719e157e24 into 764e270e3a9847b8806bffdf4bcb27de(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:52:57,162 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:57,162 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/C, priority=13, startTime=1731372776483; duration=0sec 2024-11-12T00:52:57,164 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:57,164 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:C 2024-11-12T00:52:57,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1751959ecadf419bbe5640295c8b60cc, entries=150, sequenceid=295, filesize=12.0 K 2024-11-12T00:52:57,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/4b9ce6f3b9a5457fa2df5a668518d29f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/4b9ce6f3b9a5457fa2df5a668518d29f 2024-11-12T00:52:57,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372837186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/4b9ce6f3b9a5457fa2df5a668518d29f, entries=150, sequenceid=295, filesize=12.0 K 2024-11-12T00:52:57,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8f6f5fa6d51a99bb28036e719e157e24 in 708ms, sequenceid=295, compaction requested=false 2024-11-12T00:52:57,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:57,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T00:52:57,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:52:57,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:57,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:52:57,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:57,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:52:57,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:57,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/3fcd15243c124603883aa882bb189ffb is 50, key is test_row_0/A:col10/1731372777200/Put/seqid=0 2024-11-12T00:52:57,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372837245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741870_1046 (size=14741) 2024-11-12T00:52:57,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372837251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/3fcd15243c124603883aa882bb189ffb 2024-11-12T00:52:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372837256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372837253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:57,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:57,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:57,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:57,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/5f8a547b0a2446f485358578b5ebb277 is 50, key is test_row_0/B:col10/1731372777200/Put/seqid=0 2024-11-12T00:52:57,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741871_1047 (size=12301) 2024-11-12T00:52:57,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372837365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372837376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372837376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372837376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:57,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:57,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:57,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:57,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372837576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372837586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372837586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372837589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:57,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:57,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:57,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:57,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372837705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/5f8a547b0a2446f485358578b5ebb277 2024-11-12T00:52:57,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/44192a7f65794bdb86dcb7e54fae54b2 is 50, key is test_row_0/C:col10/1731372777200/Put/seqid=0 2024-11-12T00:52:57,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741872_1048 (size=12301) 2024-11-12T00:52:57,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/44192a7f65794bdb86dcb7e54fae54b2 2024-11-12T00:52:57,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:57,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:57,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:57,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:57,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/3fcd15243c124603883aa882bb189ffb as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3fcd15243c124603883aa882bb189ffb 2024-11-12T00:52:57,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3fcd15243c124603883aa882bb189ffb, entries=200, sequenceid=312, filesize=14.4 K 2024-11-12T00:52:57,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/5f8a547b0a2446f485358578b5ebb277 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/5f8a547b0a2446f485358578b5ebb277 2024-11-12T00:52:57,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/5f8a547b0a2446f485358578b5ebb277, entries=150, sequenceid=312, filesize=12.0 K 2024-11-12T00:52:57,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/44192a7f65794bdb86dcb7e54fae54b2 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/44192a7f65794bdb86dcb7e54fae54b2 2024-11-12T00:52:57,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/44192a7f65794bdb86dcb7e54fae54b2, entries=150, sequenceid=312, filesize=12.0 K 2024-11-12T00:52:57,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8f6f5fa6d51a99bb28036e719e157e24 in 602ms, sequenceid=312, compaction requested=true 2024-11-12T00:52:57,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:57,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:52:57,805 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:52:57,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:57,806 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:52:57,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:52:57,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:57,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:52:57,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:52:57,807 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:52:57,807 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/A is initiating minor compaction (all files) 2024-11-12T00:52:57,807 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/A in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:57,808 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/56735bb1fd804a8f93ca0c97778b5fb1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/24c538ec2b00491894771b75fc8ff455, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3fcd15243c124603883aa882bb189ffb] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=38.7 K 2024-11-12T00:52:57,808 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:52:57,808 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/B is initiating minor compaction (all files) 2024-11-12T00:52:57,808 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/B in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:57,808 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/479bc5c78b2542619d5c699418935363, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1751959ecadf419bbe5640295c8b60cc, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/5f8a547b0a2446f485358578b5ebb277] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=36.3 K 2024-11-12T00:52:57,809 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56735bb1fd804a8f93ca0c97778b5fb1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731372775769 2024-11-12T00:52:57,809 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 479bc5c78b2542619d5c699418935363, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731372775769 2024-11-12T00:52:57,810 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24c538ec2b00491894771b75fc8ff455, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731372776484 2024-11-12T00:52:57,811 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fcd15243c124603883aa882bb189ffb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731372776521 2024-11-12T00:52:57,811 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 1751959ecadf419bbe5640295c8b60cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731372776484 2024-11-12T00:52:57,812 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f8a547b0a2446f485358578b5ebb277, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731372776521 2024-11-12T00:52:57,839 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:52:57,840 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/8ca8536fb08e48f3a46a112d0a73c90f is 50, key is test_row_0/B:col10/1731372777200/Put/seqid=0 2024-11-12T00:52:57,846 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:52:57,847 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/a423238bb7ba4886b4c55162cbd26954 is 50, key is test_row_0/A:col10/1731372777200/Put/seqid=0 2024-11-12T00:52:57,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741873_1049 (size=12711) 2024-11-12T00:52:57,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741874_1050 (size=12711) 2024-11-12T00:52:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:52:57,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T00:52:57,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:52:57,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:57,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:52:57,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:57,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:52:57,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:57,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/9c9c498cc4084479a4a1fb17d456806d is 50, key is test_row_0/A:col10/1731372777227/Put/seqid=0 2024-11-12T00:52:57,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:57,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:57,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:57,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:57,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:57,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372837915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372837917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372837918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741875_1051 (size=14741) 2024-11-12T00:52:57,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:57,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372837931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:57,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:57,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:57,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T00:52:58,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372838035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372838036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372838035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:58,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:58,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372838046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:58,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:58,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:58,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:58,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:58,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:58,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:58,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:58,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:58,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:58,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372838245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:58,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372838246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372838246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:58,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372838255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:58,278 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/8ca8536fb08e48f3a46a112d0a73c90f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/8ca8536fb08e48f3a46a112d0a73c90f 2024-11-12T00:52:58,296 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/B of 8f6f5fa6d51a99bb28036e719e157e24 into 8ca8536fb08e48f3a46a112d0a73c90f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:52:58,296 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:58,296 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/B, priority=13, startTime=1731372777805; duration=0sec 2024-11-12T00:52:58,297 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:52:58,297 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:B 2024-11-12T00:52:58,297 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/a423238bb7ba4886b4c55162cbd26954 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/a423238bb7ba4886b4c55162cbd26954 2024-11-12T00:52:58,297 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:52:58,300 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:52:58,300 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/C is initiating minor compaction (all files) 2024-11-12T00:52:58,301 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/C in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:58,302 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/764e270e3a9847b8806bffdf4bcb27de, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/4b9ce6f3b9a5457fa2df5a668518d29f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/44192a7f65794bdb86dcb7e54fae54b2] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=36.3 K 2024-11-12T00:52:58,302 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 764e270e3a9847b8806bffdf4bcb27de, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731372775769 2024-11-12T00:52:58,303 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b9ce6f3b9a5457fa2df5a668518d29f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731372776484 2024-11-12T00:52:58,304 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 44192a7f65794bdb86dcb7e54fae54b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731372776521 2024-11-12T00:52:58,315 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/A of 8f6f5fa6d51a99bb28036e719e157e24 into a423238bb7ba4886b4c55162cbd26954(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:52:58,315 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:58,316 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/A, priority=13, startTime=1731372777805; duration=0sec 2024-11-12T00:52:58,316 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:58,316 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:A 2024-11-12T00:52:58,317 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#C#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:52:58,318 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/11765aa085ce4491bc4aefd93f618bae is 50, key is test_row_0/C:col10/1731372777200/Put/seqid=0 2024-11-12T00:52:58,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741876_1052 (size=12711) 2024-11-12T00:52:58,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/9c9c498cc4084479a4a1fb17d456806d 2024-11-12T00:52:58,349 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/11765aa085ce4491bc4aefd93f618bae as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/11765aa085ce4491bc4aefd93f618bae 2024-11-12T00:52:58,363 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/C of 8f6f5fa6d51a99bb28036e719e157e24 into 11765aa085ce4491bc4aefd93f618bae(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:52:58,363 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:58,363 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/C, priority=13, startTime=1731372777806; duration=0sec 2024-11-12T00:52:58,364 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:52:58,364 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:C 2024-11-12T00:52:58,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3782a7daae9847fd8ce7885be1304b72 is 50, key is test_row_0/B:col10/1731372777227/Put/seqid=0 2024-11-12T00:52:58,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741877_1053 (size=12301) 2024-11-12T00:52:58,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:58,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:58,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:58,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:58,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:58,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:58,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:58,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:58,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:58,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372838555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:58,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:58,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372838555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:58,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:58,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:58,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372838566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:58,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372838566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:58,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:58,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:58,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:58,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:58,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:58,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:58,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372838715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:58,721 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 8f6f5fa6d51a99bb28036e719e157e24 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:52:58,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:58,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:58,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3782a7daae9847fd8ce7885be1304b72 2024-11-12T00:52:58,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/bf21d57b9c77488e92d9d163abab33aa is 50, key is test_row_0/C:col10/1731372777227/Put/seqid=0 2024-11-12T00:52:58,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741878_1054 (size=12301) 2024-11-12T00:52:58,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:58,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:58,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:58,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:58,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:59,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:59,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:59,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:59,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:59,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:59,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:59,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:59,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372839066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:59,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372839066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:59,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372839075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:59,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:52:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372839086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:52:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:52:59,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:52:59,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:59,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:59,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:52:59,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:59,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:59,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:59,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:52:59,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/bf21d57b9c77488e92d9d163abab33aa 2024-11-12T00:52:59,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/9c9c498cc4084479a4a1fb17d456806d as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/9c9c498cc4084479a4a1fb17d456806d 2024-11-12T00:52:59,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/9c9c498cc4084479a4a1fb17d456806d, entries=200, sequenceid=331, filesize=14.4 K 2024-11-12T00:52:59,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3782a7daae9847fd8ce7885be1304b72 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3782a7daae9847fd8ce7885be1304b72 2024-11-12T00:52:59,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3782a7daae9847fd8ce7885be1304b72, entries=150, sequenceid=331, filesize=12.0 K 2024-11-12T00:52:59,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/bf21d57b9c77488e92d9d163abab33aa as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/bf21d57b9c77488e92d9d163abab33aa 2024-11-12T00:52:59,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/bf21d57b9c77488e92d9d163abab33aa, entries=150, sequenceid=331, filesize=12.0 K 2024-11-12T00:52:59,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8f6f5fa6d51a99bb28036e719e157e24 in 1371ms, sequenceid=331, compaction requested=false 2024-11-12T00:52:59,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:52:59,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T00:52:59,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:52:59,311 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T00:52:59,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:52:59,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:59,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:52:59,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:59,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:52:59,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:52:59,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/ce2b925266d04a248d63fb0377bc02cc is 50, key is test_row_0/A:col10/1731372777914/Put/seqid=0 2024-11-12T00:52:59,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741879_1055 (size=12301) 2024-11-12T00:52:59,738 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/ce2b925266d04a248d63fb0377bc02cc 2024-11-12T00:52:59,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/4ef7bd8b70804f379ae33280392e735e is 50, key is test_row_0/B:col10/1731372777914/Put/seqid=0 2024-11-12T00:52:59,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741880_1056 (size=12301) 2024-11-12T00:52:59,782 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/4ef7bd8b70804f379ae33280392e735e 2024-11-12T00:52:59,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/b3470746cae64bb6a128f2f996dc2ba6 is 50, key is test_row_0/C:col10/1731372777914/Put/seqid=0 2024-11-12T00:52:59,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741881_1057 (size=12301) 2024-11-12T00:53:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:00,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:00,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372840127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731372780105, max timestamp=1731372780131Num uniques 859; 2024-11-12T00:53:00,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731372780105, max timestamp=1731372780131Num uniques 859; 2024-11-12T00:53:00,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731372780105, max timestamp=1731372780131Num uniques 859; 2024-11-12T00:53:00,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T00:53:00,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T00:53:00,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T00:53:00,215 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/b3470746cae64bb6a128f2f996dc2ba6 2024-11-12T00:53:00,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372840210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372840210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372840211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/ce2b925266d04a248d63fb0377bc02cc as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/ce2b925266d04a248d63fb0377bc02cc 2024-11-12T00:53:00,240 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/ce2b925266d04a248d63fb0377bc02cc, entries=150, sequenceid=347, filesize=12.0 K 2024-11-12T00:53:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/4ef7bd8b70804f379ae33280392e735e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4ef7bd8b70804f379ae33280392e735e 2024-11-12T00:53:00,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372840236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,252 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4ef7bd8b70804f379ae33280392e735e, entries=150, sequenceid=347, filesize=12.0 K 2024-11-12T00:53:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/b3470746cae64bb6a128f2f996dc2ba6 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/b3470746cae64bb6a128f2f996dc2ba6 2024-11-12T00:53:00,266 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/b3470746cae64bb6a128f2f996dc2ba6, entries=150, sequenceid=347, filesize=12.0 K 2024-11-12T00:53:00,268 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 8f6f5fa6d51a99bb28036e719e157e24 in 957ms, sequenceid=347, compaction requested=true 2024-11-12T00:53:00,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:00,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:00,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-12T00:53:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-12T00:53:00,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-12T00:53:00,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 6.4550 sec 2024-11-12T00:53:00,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 6.4680 sec 2024-11-12T00:53:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:00,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=120.76 KB heapSize=335.77 KB 2024-11-12T00:53:00,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:00,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:00,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:00,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:00,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:00,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:00,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/513e218c2c844ea596a968e6a497fab3 is 50, key is test_row_0/A:col10/1731372780330/Put/seqid=0 2024-11-12T00:53:00,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741882_1058 (size=12301) 2024-11-12T00:53:00,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/513e218c2c844ea596a968e6a497fab3 2024-11-12T00:53:00,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372840363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/4791309658d4440f8721d93b8258d60d is 50, key is test_row_0/B:col10/1731372780330/Put/seqid=0 2024-11-12T00:53:00,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372840371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372840375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741883_1059 (size=12301) 2024-11-12T00:53:00,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/4791309658d4440f8721d93b8258d60d 2024-11-12T00:53:00,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/9ff6a8b03fd34e579887890c3eee54b9 is 50, key is test_row_0/C:col10/1731372780330/Put/seqid=0 2024-11-12T00:53:00,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741884_1060 (size=12301) 2024-11-12T00:53:00,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/9ff6a8b03fd34e579887890c3eee54b9 2024-11-12T00:53:00,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/513e218c2c844ea596a968e6a497fab3 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/513e218c2c844ea596a968e6a497fab3 2024-11-12T00:53:00,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/513e218c2c844ea596a968e6a497fab3, entries=150, sequenceid=381, filesize=12.0 K 2024-11-12T00:53:00,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/4791309658d4440f8721d93b8258d60d as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4791309658d4440f8721d93b8258d60d 2024-11-12T00:53:00,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372840458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4791309658d4440f8721d93b8258d60d, entries=150, sequenceid=381, filesize=12.0 K 2024-11-12T00:53:00,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/9ff6a8b03fd34e579887890c3eee54b9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9ff6a8b03fd34e579887890c3eee54b9 2024-11-12T00:53:00,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9ff6a8b03fd34e579887890c3eee54b9, entries=150, sequenceid=381, filesize=12.0 K 2024-11-12T00:53:00,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 8f6f5fa6d51a99bb28036e719e157e24 in 159ms, sequenceid=381, compaction requested=true 2024-11-12T00:53:00,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:00,493 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:00,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:53:00,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:00,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:53:00,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:00,493 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:00,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:53:00,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:00,495 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52054 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:00,495 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/A is initiating minor compaction (all files) 2024-11-12T00:53:00,495 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/A in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:00,495 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/a423238bb7ba4886b4c55162cbd26954, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/9c9c498cc4084479a4a1fb17d456806d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/ce2b925266d04a248d63fb0377bc02cc, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/513e218c2c844ea596a968e6a497fab3] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=50.8 K 2024-11-12T00:53:00,496 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:00,496 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/B is initiating minor compaction (all files) 2024-11-12T00:53:00,496 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/B in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:00,496 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/8ca8536fb08e48f3a46a112d0a73c90f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3782a7daae9847fd8ce7885be1304b72, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4ef7bd8b70804f379ae33280392e735e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4791309658d4440f8721d93b8258d60d] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=48.5 K 2024-11-12T00:53:00,497 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting a423238bb7ba4886b4c55162cbd26954, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731372776521 2024-11-12T00:53:00,498 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ca8536fb08e48f3a46a112d0a73c90f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731372776521 2024-11-12T00:53:00,498 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c9c498cc4084479a4a1fb17d456806d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731372777227 2024-11-12T00:53:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:00,498 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 3782a7daae9847fd8ce7885be1304b72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731372777227 2024-11-12T00:53:00,499 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce2b925266d04a248d63fb0377bc02cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731372777911 2024-11-12T00:53:00,499 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ef7bd8b70804f379ae33280392e735e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731372777911 2024-11-12T00:53:00,499 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 513e218c2c844ea596a968e6a497fab3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731372780172 2024-11-12T00:53:00,500 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 4791309658d4440f8721d93b8258d60d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731372780172 2024-11-12T00:53:00,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T00:53:00,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:00,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:00,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:00,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:00,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:00,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:00,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/f147feb204004f139530262d945fa9d9 is 50, key is test_row_0/A:col10/1731372780354/Put/seqid=0 2024-11-12T00:53:00,538 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#B#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:00,539 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/6745d25257a640b69de7552ae95e9feb is 50, key is test_row_0/B:col10/1731372780330/Put/seqid=0 2024-11-12T00:53:00,539 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#A#compaction#50 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:00,540 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/c0b90393ea1f4a768c17aa72647d9d5f is 50, key is test_row_0/A:col10/1731372780330/Put/seqid=0 2024-11-12T00:53:00,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741885_1061 (size=12301) 2024-11-12T00:53:00,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372840538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372840542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372840543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741887_1063 (size=12847) 2024-11-12T00:53:00,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741886_1062 (size=12847) 2024-11-12T00:53:00,568 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/c0b90393ea1f4a768c17aa72647d9d5f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/c0b90393ea1f4a768c17aa72647d9d5f 2024-11-12T00:53:00,577 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/A of 8f6f5fa6d51a99bb28036e719e157e24 into c0b90393ea1f4a768c17aa72647d9d5f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:00,577 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:00,577 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/A, priority=12, startTime=1731372780493; duration=0sec 2024-11-12T00:53:00,577 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:00,578 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:A 2024-11-12T00:53:00,578 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:00,579 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:00,580 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/C is initiating minor compaction (all files) 2024-11-12T00:53:00,580 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/C in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:00,580 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/11765aa085ce4491bc4aefd93f618bae, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/bf21d57b9c77488e92d9d163abab33aa, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/b3470746cae64bb6a128f2f996dc2ba6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9ff6a8b03fd34e579887890c3eee54b9] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=48.5 K 2024-11-12T00:53:00,580 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11765aa085ce4491bc4aefd93f618bae, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731372776521 2024-11-12T00:53:00,581 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf21d57b9c77488e92d9d163abab33aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731372777227 2024-11-12T00:53:00,581 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3470746cae64bb6a128f2f996dc2ba6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731372777911 2024-11-12T00:53:00,582 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ff6a8b03fd34e579887890c3eee54b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731372780172 2024-11-12T00:53:00,595 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:00,595 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/6e9bfc64a77b43fa9fc2560ba15e0c9e is 50, key is test_row_0/C:col10/1731372780330/Put/seqid=0 2024-11-12T00:53:00,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741888_1064 (size=12847) 2024-11-12T00:53:00,615 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/6e9bfc64a77b43fa9fc2560ba15e0c9e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/6e9bfc64a77b43fa9fc2560ba15e0c9e 2024-11-12T00:53:00,624 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/C of 8f6f5fa6d51a99bb28036e719e157e24 into 6e9bfc64a77b43fa9fc2560ba15e0c9e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:00,624 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:00,624 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/C, priority=12, startTime=1731372780493; duration=0sec 2024-11-12T00:53:00,624 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:00,624 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:C 2024-11-12T00:53:00,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372840656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372840656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372840656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372840735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,740 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 8f6f5fa6d51a99bb28036e719e157e24 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372840766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372840865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372840866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372840866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:00,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:00,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:00,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/f147feb204004f139530262d945fa9d9 2024-11-12T00:53:00,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/423491a5362b415f9f10f28762937467 is 50, key is test_row_0/B:col10/1731372780354/Put/seqid=0 2024-11-12T00:53:00,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741889_1065 (size=12301) 2024-11-12T00:53:00,964 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/6745d25257a640b69de7552ae95e9feb as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/6745d25257a640b69de7552ae95e9feb 2024-11-12T00:53:00,972 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/B of 8f6f5fa6d51a99bb28036e719e157e24 into 6745d25257a640b69de7552ae95e9feb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:00,973 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:00,973 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/B, priority=12, startTime=1731372780493; duration=0sec 2024-11-12T00:53:00,973 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:00,973 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:B 2024-11-12T00:53:01,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372841175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:01,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:01,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372841186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:01,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372841186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:01,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:01,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372841276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:01,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/423491a5362b415f9f10f28762937467 2024-11-12T00:53:01,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/46aeea6997684ce5887bbd0e5320aedc is 50, key is test_row_0/C:col10/1731372780354/Put/seqid=0 2024-11-12T00:53:01,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741890_1066 (size=12301) 2024-11-12T00:53:01,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/46aeea6997684ce5887bbd0e5320aedc 2024-11-12T00:53:01,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/f147feb204004f139530262d945fa9d9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f147feb204004f139530262d945fa9d9 2024-11-12T00:53:01,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f147feb204004f139530262d945fa9d9, entries=150, sequenceid=394, filesize=12.0 K 2024-11-12T00:53:01,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/423491a5362b415f9f10f28762937467 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/423491a5362b415f9f10f28762937467 2024-11-12T00:53:01,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/423491a5362b415f9f10f28762937467, entries=150, sequenceid=394, filesize=12.0 K 2024-11-12T00:53:01,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/46aeea6997684ce5887bbd0e5320aedc as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/46aeea6997684ce5887bbd0e5320aedc 2024-11-12T00:53:01,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/46aeea6997684ce5887bbd0e5320aedc, entries=150, sequenceid=394, filesize=12.0 K 2024-11-12T00:53:01,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8f6f5fa6d51a99bb28036e719e157e24 in 923ms, sequenceid=394, compaction requested=false 2024-11-12T00:53:01,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:01,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T00:53:01,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:01,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:01,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372780536, max timestamp=1731372781697Num uniques 859; 2024-11-12T00:53:01,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:53:01,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372780536, max timestamp=1731372781697Num uniques 859; 2024-11-12T00:53:01,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:53:01,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:01,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:01,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:01,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:01,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/80fff96ae6924dbe81af67955b725fef is 50, key is test_row_0/A:col10/1731372781697/Put/seqid=0 2024-11-12T00:53:01,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741891_1067 (size=12301) 2024-11-12T00:53:01,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/80fff96ae6924dbe81af67955b725fef 2024-11-12T00:53:01,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/313358ba9bd7491cb1e1f48776a0fe93 is 50, key is test_row_0/B:col10/1731372781697/Put/seqid=0 2024-11-12T00:53:01,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731372781723, max timestamp=1731372781737Num uniques 859; 2024-11-12T00:53:01,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731372781723, max timestamp=1731372781737Num uniques 859; 2024-11-12T00:53:01,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731372781723, max timestamp=1731372781737Num uniques 859; 2024-11-12T00:53:01,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T00:53:01,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T00:53:01,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T00:53:01,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741892_1068 (size=12301) 2024-11-12T00:53:01,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/313358ba9bd7491cb1e1f48776a0fe93 2024-11-12T00:53:01,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372841773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:01,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/ecb3a955add048fd9c0c4dc401cd1a5f is 50, key is test_row_0/C:col10/1731372781697/Put/seqid=0 2024-11-12T00:53:01,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372841780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372841780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:01,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741893_1069 (size=12301) 2024-11-12T00:53:01,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/ecb3a955add048fd9c0c4dc401cd1a5f 2024-11-12T00:53:01,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/80fff96ae6924dbe81af67955b725fef as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/80fff96ae6924dbe81af67955b725fef 2024-11-12T00:53:01,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/80fff96ae6924dbe81af67955b725fef, entries=150, sequenceid=418, filesize=12.0 K 2024-11-12T00:53:01,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/313358ba9bd7491cb1e1f48776a0fe93 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/313358ba9bd7491cb1e1f48776a0fe93 2024-11-12T00:53:01,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/313358ba9bd7491cb1e1f48776a0fe93, entries=150, sequenceid=418, filesize=12.0 K 2024-11-12T00:53:01,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/ecb3a955add048fd9c0c4dc401cd1a5f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ecb3a955add048fd9c0c4dc401cd1a5f 2024-11-12T00:53:01,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ecb3a955add048fd9c0c4dc401cd1a5f, entries=150, sequenceid=418, filesize=12.0 K 2024-11-12T00:53:01,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=134.18 KB/137400 for 8f6f5fa6d51a99bb28036e719e157e24 in 135ms, sequenceid=418, compaction requested=true 2024-11-12T00:53:01,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:01,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:53:01,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:01,835 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:01,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:53:01,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:01,835 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:01,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:53:01,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:01,837 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:01,837 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/A is initiating minor compaction (all files) 2024-11-12T00:53:01,837 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/A in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:01,837 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/c0b90393ea1f4a768c17aa72647d9d5f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f147feb204004f139530262d945fa9d9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/80fff96ae6924dbe81af67955b725fef] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=36.6 K 2024-11-12T00:53:01,838 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0b90393ea1f4a768c17aa72647d9d5f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731372780172 2024-11-12T00:53:01,838 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:01,838 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/B is initiating minor compaction (all files) 2024-11-12T00:53:01,838 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/B in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:01,838 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting f147feb204004f139530262d945fa9d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731372780334 2024-11-12T00:53:01,838 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/6745d25257a640b69de7552ae95e9feb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/423491a5362b415f9f10f28762937467, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/313358ba9bd7491cb1e1f48776a0fe93] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=36.6 K 2024-11-12T00:53:01,839 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80fff96ae6924dbe81af67955b725fef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731372780536 2024-11-12T00:53:01,839 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 6745d25257a640b69de7552ae95e9feb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731372780172 2024-11-12T00:53:01,839 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 423491a5362b415f9f10f28762937467, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731372780334 2024-11-12T00:53:01,840 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 313358ba9bd7491cb1e1f48776a0fe93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731372780536 2024-11-12T00:53:01,852 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#A#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:01,853 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/7db5e14c43d444b4816ecb9e07984fc2 is 50, key is test_row_0/A:col10/1731372781697/Put/seqid=0 2024-11-12T00:53:01,859 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:01,860 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/bfb677e7db6c46b29db9b099b4fbace8 is 50, key is test_row_0/B:col10/1731372781697/Put/seqid=0 2024-11-12T00:53:01,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741894_1070 (size=12949) 2024-11-12T00:53:01,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741895_1071 (size=12949) 2024-11-12T00:53:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:01,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-12T00:53:01,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:01,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:01,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:01,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:01,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:01,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:01,896 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/7db5e14c43d444b4816ecb9e07984fc2 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/7db5e14c43d444b4816ecb9e07984fc2 2024-11-12T00:53:01,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/63744c4a12904953b75c6fb7251dc13b is 50, key is test_row_0/A:col10/1731372781769/Put/seqid=0 2024-11-12T00:53:01,904 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/A of 8f6f5fa6d51a99bb28036e719e157e24 into 7db5e14c43d444b4816ecb9e07984fc2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:01,904 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:01,904 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/A, priority=13, startTime=1731372781835; duration=0sec 2024-11-12T00:53:01,904 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:01,904 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:A 2024-11-12T00:53:01,904 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:01,905 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:01,906 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/C is initiating minor compaction (all files) 2024-11-12T00:53:01,906 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/C in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:01,906 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/6e9bfc64a77b43fa9fc2560ba15e0c9e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/46aeea6997684ce5887bbd0e5320aedc, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ecb3a955add048fd9c0c4dc401cd1a5f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=36.6 K 2024-11-12T00:53:01,913 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e9bfc64a77b43fa9fc2560ba15e0c9e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731372780172 2024-11-12T00:53:01,914 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46aeea6997684ce5887bbd0e5320aedc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731372780334 2024-11-12T00:53:01,914 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecb3a955add048fd9c0c4dc401cd1a5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731372780536 2024-11-12T00:53:01,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741896_1072 (size=14741) 2024-11-12T00:53:01,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372841914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:01,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/63744c4a12904953b75c6fb7251dc13b 2024-11-12T00:53:01,925 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#C#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:01,926 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/5a63fe0006034bcbbd140a14f0d01259 is 50, key is test_row_0/C:col10/1731372781697/Put/seqid=0 2024-11-12T00:53:01,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372841922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:01,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:01,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372841923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:01,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/d89c7cac4ed740d9b932f74e302a2ed6 is 50, key is test_row_0/B:col10/1731372781769/Put/seqid=0 2024-11-12T00:53:01,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741897_1073 (size=12949) 2024-11-12T00:53:01,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741898_1074 (size=12301) 2024-11-12T00:53:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T00:53:02,006 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:02,007 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-12T00:53:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T00:53:02,009 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:02,010 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:02,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:02,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372842025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:02,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372842035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372842035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T00:53:02,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T00:53:02,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:02,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:02,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:02,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:02,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:02,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372842235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:02,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372842245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372842246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:02,288 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/bfb677e7db6c46b29db9b099b4fbace8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/bfb677e7db6c46b29db9b099b4fbace8 2024-11-12T00:53:02,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372842285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,290 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 8f6f5fa6d51a99bb28036e719e157e24 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:02,296 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/B of 8f6f5fa6d51a99bb28036e719e157e24 into bfb677e7db6c46b29db9b099b4fbace8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:02,296 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:02,296 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/B, priority=13, startTime=1731372781835; duration=0sec 2024-11-12T00:53:02,296 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:02,297 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:B 2024-11-12T00:53:02,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T00:53:02,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:02,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:02,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:02,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:02,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T00:53:02,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/d89c7cac4ed740d9b932f74e302a2ed6 2024-11-12T00:53:02,355 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/5a63fe0006034bcbbd140a14f0d01259 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/5a63fe0006034bcbbd140a14f0d01259 2024-11-12T00:53:02,362 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/C of 8f6f5fa6d51a99bb28036e719e157e24 into 5a63fe0006034bcbbd140a14f0d01259(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:02,362 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:02,362 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/C, priority=13, startTime=1731372781835; duration=0sec 2024-11-12T00:53:02,363 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:02,363 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:C 2024-11-12T00:53:02,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/9ae217f47fe9456a8d0a10656657b822 is 50, key is test_row_0/C:col10/1731372781769/Put/seqid=0 2024-11-12T00:53:02,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741899_1075 (size=12301) 2024-11-12T00:53:02,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/9ae217f47fe9456a8d0a10656657b822 2024-11-12T00:53:02,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/63744c4a12904953b75c6fb7251dc13b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/63744c4a12904953b75c6fb7251dc13b 2024-11-12T00:53:02,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/63744c4a12904953b75c6fb7251dc13b, entries=200, sequenceid=452, filesize=14.4 K 2024-11-12T00:53:02,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/d89c7cac4ed740d9b932f74e302a2ed6 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/d89c7cac4ed740d9b932f74e302a2ed6 2024-11-12T00:53:02,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/d89c7cac4ed740d9b932f74e302a2ed6, entries=150, sequenceid=452, filesize=12.0 K 2024-11-12T00:53:02,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/9ae217f47fe9456a8d0a10656657b822 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9ae217f47fe9456a8d0a10656657b822 2024-11-12T00:53:02,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9ae217f47fe9456a8d0a10656657b822, entries=150, sequenceid=452, filesize=12.0 K 2024-11-12T00:53:02,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 8f6f5fa6d51a99bb28036e719e157e24 in 520ms, sequenceid=452, compaction requested=false 2024-11-12T00:53:02,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:02,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T00:53:02,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:02,470 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T00:53:02,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:02,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:02,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:02,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:02,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:02,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:02,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/7b27e09da08d4902aef3504639ade6e8 is 50, key is test_row_0/A:col10/1731372781909/Put/seqid=0 2024-11-12T00:53:02,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741900_1076 (size=12301) 2024-11-12T00:53:02,496 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/7b27e09da08d4902aef3504639ade6e8 2024-11-12T00:53:02,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3022c7a4f4ef4160b08600cc888df906 is 50, key is test_row_0/B:col10/1731372781909/Put/seqid=0 2024-11-12T00:53:02,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741901_1077 (size=12301) 2024-11-12T00:53:02,521 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3022c7a4f4ef4160b08600cc888df906 2024-11-12T00:53:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/f76850fbf3ce40a997ecad47b970f357 is 50, key is test_row_0/C:col10/1731372781909/Put/seqid=0 2024-11-12T00:53:02,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741902_1078 (size=12301) 2024-11-12T00:53:02,547 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/f76850fbf3ce40a997ecad47b970f357 2024-11-12T00:53:02,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:02,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:02,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/7b27e09da08d4902aef3504639ade6e8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/7b27e09da08d4902aef3504639ade6e8 2024-11-12T00:53:02,566 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/7b27e09da08d4902aef3504639ade6e8, entries=150, sequenceid=465, filesize=12.0 K 2024-11-12T00:53:02,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3022c7a4f4ef4160b08600cc888df906 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3022c7a4f4ef4160b08600cc888df906 2024-11-12T00:53:02,578 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3022c7a4f4ef4160b08600cc888df906, entries=150, sequenceid=465, filesize=12.0 K 2024-11-12T00:53:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/f76850fbf3ce40a997ecad47b970f357 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/f76850fbf3ce40a997ecad47b970f357 2024-11-12T00:53:02,588 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/f76850fbf3ce40a997ecad47b970f357, entries=150, sequenceid=465, filesize=12.0 K 2024-11-12T00:53:02,590 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=100.63 KB/103050 for 8f6f5fa6d51a99bb28036e719e157e24 in 120ms, sequenceid=465, compaction requested=true 2024-11-12T00:53:02,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:02,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:02,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-12T00:53:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-12T00:53:02,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-12T00:53:02,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 582 msec 2024-11-12T00:53:02,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T00:53:02,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:02,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:02,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:02,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:02,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:02,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:02,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 589 msec 2024-11-12T00:53:02,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/6d8a8fb5e8ac4615a20c6de15a9d3ea1 is 50, key is test_row_0/A:col10/1731372782594/Put/seqid=0 2024-11-12T00:53:02,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741903_1079 (size=17181) 2024-11-12T00:53:02,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372842618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372842622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:02,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372842628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T00:53:02,636 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:02,636 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-12T00:53:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T00:53:02,640 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:02,641 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:02,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:02,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372842736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372842736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T00:53:02,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372842746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:02,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T00:53:02,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:02,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:02,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:02,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:02,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:02,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T00:53:02,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:02,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:02,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:02,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:02,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T00:53:02,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372842956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372842956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:02,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372842956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:02,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:02,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:03,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/6d8a8fb5e8ac4615a20c6de15a9d3ea1 2024-11-12T00:53:03,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/4cf91fcd480c4243ab0da9900b86c08c is 50, key is test_row_0/B:col10/1731372782594/Put/seqid=0 2024-11-12T00:53:03,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741904_1080 (size=12301) 2024-11-12T00:53:03,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T00:53:03,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:03,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:03,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:03,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T00:53:03,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:03,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:03,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:03,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T00:53:03,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372843266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:03,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372843266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:03,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372843275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:03,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T00:53:03,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:03,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:03,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:03,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/4cf91fcd480c4243ab0da9900b86c08c 2024-11-12T00:53:03,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/ce52b7cdd38c486e856935d41474936b is 50, key is test_row_0/C:col10/1731372782594/Put/seqid=0 2024-11-12T00:53:03,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741905_1081 (size=12301) 2024-11-12T00:53:03,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T00:53:03,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:03,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:03,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:03,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T00:53:03,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:03,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:03,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:03,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T00:53:03,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372843776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:03,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372843776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:03,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372843785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:03,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/ce52b7cdd38c486e856935d41474936b 2024-11-12T00:53:03,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T00:53:03,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:03,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:03,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:03,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:03,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/6d8a8fb5e8ac4615a20c6de15a9d3ea1 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/6d8a8fb5e8ac4615a20c6de15a9d3ea1 2024-11-12T00:53:03,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/6d8a8fb5e8ac4615a20c6de15a9d3ea1, entries=250, sequenceid=484, filesize=16.8 K 2024-11-12T00:53:03,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/4cf91fcd480c4243ab0da9900b86c08c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4cf91fcd480c4243ab0da9900b86c08c 2024-11-12T00:53:03,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4cf91fcd480c4243ab0da9900b86c08c, entries=150, sequenceid=484, filesize=12.0 K 2024-11-12T00:53:03,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/ce52b7cdd38c486e856935d41474936b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ce52b7cdd38c486e856935d41474936b 2024-11-12T00:53:03,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ce52b7cdd38c486e856935d41474936b, entries=150, sequenceid=484, filesize=12.0 K 2024-11-12T00:53:03,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8f6f5fa6d51a99bb28036e719e157e24 in 1331ms, sequenceid=484, compaction requested=true 2024-11-12T00:53:03,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:03,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:53:03,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:03,928 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:03,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:53:03,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:03,928 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:03,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:53:03,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:03,934 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:03,934 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57172 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:03,934 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/A is initiating minor compaction (all files) 2024-11-12T00:53:03,934 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/B is initiating minor compaction (all files) 2024-11-12T00:53:03,934 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/A in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:03,934 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/B in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:03,935 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/bfb677e7db6c46b29db9b099b4fbace8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/d89c7cac4ed740d9b932f74e302a2ed6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3022c7a4f4ef4160b08600cc888df906, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4cf91fcd480c4243ab0da9900b86c08c] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=48.7 K 2024-11-12T00:53:03,935 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/7db5e14c43d444b4816ecb9e07984fc2, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/63744c4a12904953b75c6fb7251dc13b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/7b27e09da08d4902aef3504639ade6e8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/6d8a8fb5e8ac4615a20c6de15a9d3ea1] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=55.8 K 2024-11-12T00:53:03,935 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting bfb677e7db6c46b29db9b099b4fbace8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731372780536 2024-11-12T00:53:03,935 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7db5e14c43d444b4816ecb9e07984fc2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731372780536 2024-11-12T00:53:03,936 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63744c4a12904953b75c6fb7251dc13b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731372781766 2024-11-12T00:53:03,936 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting d89c7cac4ed740d9b932f74e302a2ed6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731372781769 2024-11-12T00:53:03,937 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b27e09da08d4902aef3504639ade6e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731372781894 2024-11-12T00:53:03,939 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 3022c7a4f4ef4160b08600cc888df906, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731372781894 2024-11-12T00:53:03,939 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d8a8fb5e8ac4615a20c6de15a9d3ea1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731372782578 2024-11-12T00:53:03,941 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cf91fcd480c4243ab0da9900b86c08c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731372782586 2024-11-12T00:53:03,962 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:03,963 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/f9adc5fe556042bd9d52e3896ff710ae is 50, key is test_row_0/A:col10/1731372782594/Put/seqid=0 2024-11-12T00:53:03,972 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:03,972 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/1c2ee1ee330a4d839d700cced0fdbc12 is 50, key is test_row_0/B:col10/1731372782594/Put/seqid=0 2024-11-12T00:53:03,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741906_1082 (size=13085) 2024-11-12T00:53:03,986 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/f9adc5fe556042bd9d52e3896ff710ae as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f9adc5fe556042bd9d52e3896ff710ae 2024-11-12T00:53:04,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741907_1083 (size=13085) 2024-11-12T00:53:04,022 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/A of 8f6f5fa6d51a99bb28036e719e157e24 into f9adc5fe556042bd9d52e3896ff710ae(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:04,022 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:04,022 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/A, priority=12, startTime=1731372783928; duration=0sec 2024-11-12T00:53:04,022 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:04,022 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:A 2024-11-12T00:53:04,022 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:04,026 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:04,026 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/C is initiating minor compaction (all files) 2024-11-12T00:53:04,026 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/C in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:04,026 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/5a63fe0006034bcbbd140a14f0d01259, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9ae217f47fe9456a8d0a10656657b822, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/f76850fbf3ce40a997ecad47b970f357, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ce52b7cdd38c486e856935d41474936b] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=48.7 K 2024-11-12T00:53:04,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T00:53:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:04,030 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a63fe0006034bcbbd140a14f0d01259, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731372780536 2024-11-12T00:53:04,030 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T00:53:04,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:04,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:04,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:04,030 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ae217f47fe9456a8d0a10656657b822, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731372781769 2024-11-12T00:53:04,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:04,037 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting f76850fbf3ce40a997ecad47b970f357, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731372781894 2024-11-12T00:53:04,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:04,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:04,041 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce52b7cdd38c486e856935d41474936b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731372782586 2024-11-12T00:53:04,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/4d8fa101cf8c4c71a87ef695bb3bf4a3 is 50, key is test_row_0/A:col10/1731372782624/Put/seqid=0 2024-11-12T00:53:04,068 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#C#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:04,068 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/e526ac5850924f749031294a5fe836bc is 50, key is test_row_0/C:col10/1731372782594/Put/seqid=0 2024-11-12T00:53:04,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741908_1084 (size=12301) 2024-11-12T00:53:04,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741909_1085 (size=13085) 2024-11-12T00:53:04,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:04,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:04,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372844377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:04,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:04,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:04,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:04,419 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/1c2ee1ee330a4d839d700cced0fdbc12 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1c2ee1ee330a4d839d700cced0fdbc12 2024-11-12T00:53:04,427 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/B of 8f6f5fa6d51a99bb28036e719e157e24 into 1c2ee1ee330a4d839d700cced0fdbc12(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:04,427 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:04,427 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/B, priority=12, startTime=1731372783928; duration=0sec 2024-11-12T00:53:04,427 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:04,427 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:B 2024-11-12T00:53:04,477 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/4d8fa101cf8c4c71a87ef695bb3bf4a3 2024-11-12T00:53:04,485 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/e526ac5850924f749031294a5fe836bc as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/e526ac5850924f749031294a5fe836bc 2024-11-12T00:53:04,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372844486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:04,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/98a88e1f90bf46789ad26986054a92bf is 50, key is test_row_0/B:col10/1731372782624/Put/seqid=0 2024-11-12T00:53:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:04,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:04,492 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/C of 8f6f5fa6d51a99bb28036e719e157e24 into e526ac5850924f749031294a5fe836bc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:04,492 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:04,492 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/C, priority=12, startTime=1731372783928; duration=0sec 2024-11-12T00:53:04,492 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:04,492 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:C 2024-11-12T00:53:04,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741910_1086 (size=12301) 2024-11-12T00:53:04,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372844695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:04,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T00:53:04,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:04,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45746 deadline: 1731372844785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:04,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:04,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372844786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:04,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:04,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372844786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:04,789 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_1' in 8f6f5fa6d51a99bb28036e719e157e24 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:04,789 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 8f6f5fa6d51a99bb28036e719e157e24 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:04,789 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 8f6f5fa6d51a99bb28036e719e157e24 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:04,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:04,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:04,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372844805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:04,809 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 8f6f5fa6d51a99bb28036e719e157e24 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:04,896 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/98a88e1f90bf46789ad26986054a92bf 2024-11-12T00:53:04,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/32a09dd2da614faa82e45e244466c770 is 50, key is test_row_0/C:col10/1731372782624/Put/seqid=0 2024-11-12T00:53:04,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741911_1087 (size=12301) 2024-11-12T00:53:04,914 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/32a09dd2da614faa82e45e244466c770 2024-11-12T00:53:04,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/4d8fa101cf8c4c71a87ef695bb3bf4a3 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/4d8fa101cf8c4c71a87ef695bb3bf4a3 2024-11-12T00:53:04,927 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/4d8fa101cf8c4c71a87ef695bb3bf4a3, entries=150, sequenceid=498, filesize=12.0 K 2024-11-12T00:53:04,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/98a88e1f90bf46789ad26986054a92bf as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/98a88e1f90bf46789ad26986054a92bf 2024-11-12T00:53:04,934 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/98a88e1f90bf46789ad26986054a92bf, entries=150, sequenceid=498, filesize=12.0 K 2024-11-12T00:53:04,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/32a09dd2da614faa82e45e244466c770 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/32a09dd2da614faa82e45e244466c770 2024-11-12T00:53:04,942 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/32a09dd2da614faa82e45e244466c770, entries=150, sequenceid=498, filesize=12.0 K 2024-11-12T00:53:04,943 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8f6f5fa6d51a99bb28036e719e157e24 in 914ms, sequenceid=498, compaction requested=false 2024-11-12T00:53:04,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:04,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:04,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-12T00:53:04,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-12T00:53:04,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-12T00:53:04,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3040 sec 2024-11-12T00:53:04,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.3120 sec 2024-11-12T00:53:05,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:05,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T00:53:05,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:05,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:05,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:05,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:05,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:05,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:05,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/5a7f318da2a94cf5a7f6c757fbca2902 is 50, key is test_row_0/A:col10/1731372784368/Put/seqid=0 2024-11-12T00:53:05,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741912_1088 (size=14741) 2024-11-12T00:53:05,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372845056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:05,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:05,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:05,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:05,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372845165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:05,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:05,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:05,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:05,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372845376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:05,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/5a7f318da2a94cf5a7f6c757fbca2902 2024-11-12T00:53:05,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/c963e00be1e34d72822b7ba27d673445 is 50, key is test_row_0/B:col10/1731372784368/Put/seqid=0 2024-11-12T00:53:05,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741913_1089 (size=12301) 2024-11-12T00:53:05,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/c963e00be1e34d72822b7ba27d673445 2024-11-12T00:53:05,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/0ea98005f161445eafe643bfdb51066f is 50, key is test_row_0/C:col10/1731372784368/Put/seqid=0 2024-11-12T00:53:05,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741914_1090 (size=12301) 2024-11-12T00:53:05,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/0ea98005f161445eafe643bfdb51066f 2024-11-12T00:53:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/5a7f318da2a94cf5a7f6c757fbca2902 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/5a7f318da2a94cf5a7f6c757fbca2902 2024-11-12T00:53:05,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/5a7f318da2a94cf5a7f6c757fbca2902, entries=200, sequenceid=520, filesize=14.4 K 2024-11-12T00:53:05,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/c963e00be1e34d72822b7ba27d673445 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/c963e00be1e34d72822b7ba27d673445 2024-11-12T00:53:05,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/c963e00be1e34d72822b7ba27d673445, entries=150, sequenceid=520, filesize=12.0 K 2024-11-12T00:53:05,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/0ea98005f161445eafe643bfdb51066f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/0ea98005f161445eafe643bfdb51066f 2024-11-12T00:53:05,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/0ea98005f161445eafe643bfdb51066f, entries=150, sequenceid=520, filesize=12.0 K 2024-11-12T00:53:05,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8f6f5fa6d51a99bb28036e719e157e24 in 489ms, sequenceid=520, compaction requested=true 2024-11-12T00:53:05,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:05,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:53:05,499 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:05,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:05,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:53:05,499 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:05,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:05,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:53:05,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:05,500 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:05,500 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:05,500 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/A is initiating minor compaction (all files) 2024-11-12T00:53:05,500 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/B is initiating minor compaction (all files) 2024-11-12T00:53:05,501 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/A in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:05,501 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/B in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:05,501 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1c2ee1ee330a4d839d700cced0fdbc12, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/98a88e1f90bf46789ad26986054a92bf, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/c963e00be1e34d72822b7ba27d673445] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=36.8 K 2024-11-12T00:53:05,501 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f9adc5fe556042bd9d52e3896ff710ae, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/4d8fa101cf8c4c71a87ef695bb3bf4a3, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/5a7f318da2a94cf5a7f6c757fbca2902] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=39.2 K 2024-11-12T00:53:05,501 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c2ee1ee330a4d839d700cced0fdbc12, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731372782586 2024-11-12T00:53:05,502 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9adc5fe556042bd9d52e3896ff710ae, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731372782586 2024-11-12T00:53:05,502 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 98a88e1f90bf46789ad26986054a92bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731372782616 2024-11-12T00:53:05,502 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d8fa101cf8c4c71a87ef695bb3bf4a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731372782616 2024-11-12T00:53:05,502 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting c963e00be1e34d72822b7ba27d673445, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1731372784368 2024-11-12T00:53:05,503 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a7f318da2a94cf5a7f6c757fbca2902, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1731372784354 2024-11-12T00:53:05,518 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#B#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:05,518 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/9f2d79e96d6f49aa926f4f5f7fdd13ef is 50, key is test_row_0/B:col10/1731372784368/Put/seqid=0 2024-11-12T00:53:05,531 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#A#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:05,531 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/99e2bff1a39042f2938eafe868ad4150 is 50, key is test_row_0/A:col10/1731372784368/Put/seqid=0 2024-11-12T00:53:05,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741915_1091 (size=13187) 2024-11-12T00:53:05,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741916_1092 (size=13187) 2024-11-12T00:53:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:05,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T00:53:05,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:05,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:05,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:05,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:05,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:05,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:05,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/9403c78408a246aab911e628505de394 is 50, key is test_row_0/A:col10/1731372785054/Put/seqid=0 2024-11-12T00:53:05,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741917_1093 (size=14741) 2024-11-12T00:53:05,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/9403c78408a246aab911e628505de394 2024-11-12T00:53:05,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/1d5d60ed833a4fa0a1a9e77a51cf495b is 50, key is test_row_0/B:col10/1731372785054/Put/seqid=0 2024-11-12T00:53:05,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741918_1094 (size=12301) 2024-11-12T00:53:05,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372845773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:05,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:05,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:05,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:05,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372845886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:05,946 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/9f2d79e96d6f49aa926f4f5f7fdd13ef as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/9f2d79e96d6f49aa926f4f5f7fdd13ef 2024-11-12T00:53:05,953 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/99e2bff1a39042f2938eafe868ad4150 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/99e2bff1a39042f2938eafe868ad4150 2024-11-12T00:53:05,957 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/B of 8f6f5fa6d51a99bb28036e719e157e24 into 9f2d79e96d6f49aa926f4f5f7fdd13ef(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:05,957 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:05,957 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/B, priority=13, startTime=1731372785499; duration=0sec 2024-11-12T00:53:05,957 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:05,957 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:B 2024-11-12T00:53:05,957 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:05,959 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:05,959 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/C is initiating minor compaction (all files) 2024-11-12T00:53:05,959 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/C in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:05,959 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/e526ac5850924f749031294a5fe836bc, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/32a09dd2da614faa82e45e244466c770, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/0ea98005f161445eafe643bfdb51066f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=36.8 K 2024-11-12T00:53:05,959 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting e526ac5850924f749031294a5fe836bc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1731372782586 2024-11-12T00:53:05,960 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 32a09dd2da614faa82e45e244466c770, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731372782616 2024-11-12T00:53:05,960 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ea98005f161445eafe643bfdb51066f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1731372784368 2024-11-12T00:53:05,967 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/A of 8f6f5fa6d51a99bb28036e719e157e24 into 99e2bff1a39042f2938eafe868ad4150(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:05,967 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:05,967 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/A, priority=13, startTime=1731372785499; duration=0sec 2024-11-12T00:53:05,967 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:05,967 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:A 2024-11-12T00:53:05,973 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#C#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:05,974 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/3214525920794454ae99ffa28dfdc5ce is 50, key is test_row_0/C:col10/1731372784368/Put/seqid=0 2024-11-12T00:53:05,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741919_1095 (size=13187) 2024-11-12T00:53:05,991 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/3214525920794454ae99ffa28dfdc5ce as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/3214525920794454ae99ffa28dfdc5ce 2024-11-12T00:53:05,997 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/C of 8f6f5fa6d51a99bb28036e719e157e24 into 3214525920794454ae99ffa28dfdc5ce(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:05,997 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:05,997 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/C, priority=13, startTime=1731372785499; duration=0sec 2024-11-12T00:53:05,998 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:05,998 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:C 2024-11-12T00:53:06,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372846095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:06,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/1d5d60ed833a4fa0a1a9e77a51cf495b 2024-11-12T00:53:06,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/82bc1a6387ac408b9e75b833e0e53b84 is 50, key is test_row_0/C:col10/1731372785054/Put/seqid=0 2024-11-12T00:53:06,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741920_1096 (size=12301) 2024-11-12T00:53:06,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/82bc1a6387ac408b9e75b833e0e53b84 2024-11-12T00:53:06,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/9403c78408a246aab911e628505de394 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/9403c78408a246aab911e628505de394 2024-11-12T00:53:06,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/9403c78408a246aab911e628505de394, entries=200, sequenceid=533, filesize=14.4 K 2024-11-12T00:53:06,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/1d5d60ed833a4fa0a1a9e77a51cf495b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1d5d60ed833a4fa0a1a9e77a51cf495b 2024-11-12T00:53:06,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1d5d60ed833a4fa0a1a9e77a51cf495b, entries=150, sequenceid=533, filesize=12.0 K 2024-11-12T00:53:06,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/82bc1a6387ac408b9e75b833e0e53b84 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/82bc1a6387ac408b9e75b833e0e53b84 2024-11-12T00:53:06,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/82bc1a6387ac408b9e75b833e0e53b84, entries=150, sequenceid=533, filesize=12.0 K 2024-11-12T00:53:06,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8f6f5fa6d51a99bb28036e719e157e24 in 486ms, sequenceid=533, compaction requested=false 2024-11-12T00:53:06,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:06,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T00:53:06,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:06,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:06,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:06,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:06,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:06,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:06,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/883f6d4a17aa45d391fe1705778b5e33 is 50, key is test_row_0/A:col10/1731372786406/Put/seqid=0 2024-11-12T00:53:06,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741921_1097 (size=14741) 2024-11-12T00:53:06,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372846441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:06,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372846555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:06,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:06,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:06,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372846766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:06,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:06,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:06,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:06,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372846795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:06,796 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_1' in 8f6f5fa6d51a99bb28036e719e157e24 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:06,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T00:53:06,806 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:06,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372846805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:06,806 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:06,806 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 8f6f5fa6d51a99bb28036e719e157e24 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-12T00:53:06,808 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T00:53:06,809 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:06,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:06,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/883f6d4a17aa45d391fe1705778b5e33 2024-11-12T00:53:06,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/50d208304d224c8f856f79e3657f78d5 is 50, key is test_row_0/B:col10/1731372786406/Put/seqid=0 2024-11-12T00:53:06,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741922_1098 (size=12301) 2024-11-12T00:53:06,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/50d208304d224c8f856f79e3657f78d5 2024-11-12T00:53:06,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372846836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:06,840 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 8f6f5fa6d51a99bb28036e719e157e24 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:06,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:06,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/bd8a6eaa594a44eba6f76ff754dc887f is 50, key is test_row_0/C:col10/1731372786406/Put/seqid=0 2024-11-12T00:53:06,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741923_1099 (size=12301) 2024-11-12T00:53:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T00:53:06,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T00:53:06,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:06,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:06,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:06,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:06,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:07,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372847076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:07,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T00:53:07,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:07,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:07,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:07,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:07,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T00:53:07,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/bd8a6eaa594a44eba6f76ff754dc887f 2024-11-12T00:53:07,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/883f6d4a17aa45d391fe1705778b5e33 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/883f6d4a17aa45d391fe1705778b5e33 2024-11-12T00:53:07,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/883f6d4a17aa45d391fe1705778b5e33, entries=200, sequenceid=556, filesize=14.4 K 2024-11-12T00:53:07,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/50d208304d224c8f856f79e3657f78d5 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/50d208304d224c8f856f79e3657f78d5 2024-11-12T00:53:07,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/50d208304d224c8f856f79e3657f78d5, entries=150, sequenceid=556, filesize=12.0 K 2024-11-12T00:53:07,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T00:53:07,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:07,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:07,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:07,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:07,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:07,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/bd8a6eaa594a44eba6f76ff754dc887f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/bd8a6eaa594a44eba6f76ff754dc887f 2024-11-12T00:53:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:07,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/bd8a6eaa594a44eba6f76ff754dc887f, entries=150, sequenceid=556, filesize=12.0 K 2024-11-12T00:53:07,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8f6f5fa6d51a99bb28036e719e157e24 in 868ms, sequenceid=556, compaction requested=true 2024-11-12T00:53:07,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:07,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:53:07,277 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:07,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:07,277 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:07,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:53:07,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:07,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:53:07,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:07,278 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:07,278 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:07,278 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/B is initiating minor compaction (all files) 2024-11-12T00:53:07,278 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/A is initiating minor compaction (all files) 2024-11-12T00:53:07,278 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/B in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:07,278 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/A in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:07,279 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/99e2bff1a39042f2938eafe868ad4150, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/9403c78408a246aab911e628505de394, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/883f6d4a17aa45d391fe1705778b5e33] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=41.7 K 2024-11-12T00:53:07,279 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/9f2d79e96d6f49aa926f4f5f7fdd13ef, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1d5d60ed833a4fa0a1a9e77a51cf495b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/50d208304d224c8f856f79e3657f78d5] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=36.9 K 2024-11-12T00:53:07,279 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f2d79e96d6f49aa926f4f5f7fdd13ef, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1731372784368 2024-11-12T00:53:07,279 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99e2bff1a39042f2938eafe868ad4150, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1731372784368 2024-11-12T00:53:07,279 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d5d60ed833a4fa0a1a9e77a51cf495b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1731372785030 2024-11-12T00:53:07,280 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9403c78408a246aab911e628505de394, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1731372785030 2024-11-12T00:53:07,280 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 50d208304d224c8f856f79e3657f78d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1731372785767 2024-11-12T00:53:07,280 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 883f6d4a17aa45d391fe1705778b5e33, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1731372785739 2024-11-12T00:53:07,291 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#B#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:07,292 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3e067a6ecb224dee97913d98f5b4b49f is 50, key is test_row_0/B:col10/1731372786406/Put/seqid=0 2024-11-12T00:53:07,294 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#A#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:07,294 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/12e4a441d30d49c7825953e1b5184758 is 50, key is test_row_0/A:col10/1731372786406/Put/seqid=0 2024-11-12T00:53:07,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741924_1100 (size=13289) 2024-11-12T00:53:07,330 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3e067a6ecb224dee97913d98f5b4b49f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e067a6ecb224dee97913d98f5b4b49f 2024-11-12T00:53:07,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741925_1101 (size=13289) 2024-11-12T00:53:07,336 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/12e4a441d30d49c7825953e1b5184758 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/12e4a441d30d49c7825953e1b5184758 2024-11-12T00:53:07,337 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/B of 8f6f5fa6d51a99bb28036e719e157e24 into 3e067a6ecb224dee97913d98f5b4b49f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:07,338 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:07,338 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/B, priority=13, startTime=1731372787277; duration=0sec 2024-11-12T00:53:07,338 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:07,338 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:B 2024-11-12T00:53:07,338 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:07,340 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:07,340 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/C is initiating minor compaction (all files) 2024-11-12T00:53:07,341 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/C in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:07,341 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/3214525920794454ae99ffa28dfdc5ce, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/82bc1a6387ac408b9e75b833e0e53b84, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/bd8a6eaa594a44eba6f76ff754dc887f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=36.9 K 2024-11-12T00:53:07,341 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 3214525920794454ae99ffa28dfdc5ce, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1731372784368 2024-11-12T00:53:07,342 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 82bc1a6387ac408b9e75b833e0e53b84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1731372785030 2024-11-12T00:53:07,343 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting bd8a6eaa594a44eba6f76ff754dc887f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1731372785767 2024-11-12T00:53:07,346 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/A of 8f6f5fa6d51a99bb28036e719e157e24 into 12e4a441d30d49c7825953e1b5184758(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:07,346 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:07,346 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/A, priority=13, startTime=1731372787277; duration=0sec 2024-11-12T00:53:07,346 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:07,346 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:A 2024-11-12T00:53:07,353 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#C#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:07,354 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/0697dde878d54eecb341a235c277ea34 is 50, key is test_row_0/C:col10/1731372786406/Put/seqid=0 2024-11-12T00:53:07,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741926_1102 (size=13289) 2024-11-12T00:53:07,367 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/0697dde878d54eecb341a235c277ea34 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/0697dde878d54eecb341a235c277ea34 2024-11-12T00:53:07,373 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/C of 8f6f5fa6d51a99bb28036e719e157e24 into 0697dde878d54eecb341a235c277ea34(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:07,373 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:07,373 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/C, priority=13, startTime=1731372787277; duration=0sec 2024-11-12T00:53:07,373 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:07,373 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:C 2024-11-12T00:53:07,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T00:53:07,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:07,422 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T00:53:07,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:07,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:07,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:07,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:07,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:07,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:07,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/f769e667204645c4acc9b98296956dd2 is 50, key is test_row_0/A:col10/1731372786439/Put/seqid=0 2024-11-12T00:53:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T00:53:07,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741927_1103 (size=12301) 2024-11-12T00:53:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:07,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:07,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372847651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:07,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:07,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372847755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:07,838 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/f769e667204645c4acc9b98296956dd2 2024-11-12T00:53:07,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/7e183781a9c94aa6991c0f4dce7cec80 is 50, key is test_row_0/B:col10/1731372786439/Put/seqid=0 2024-11-12T00:53:07,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741928_1104 (size=12301) 2024-11-12T00:53:07,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T00:53:07,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:07,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372847966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:08,253 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/7e183781a9c94aa6991c0f4dce7cec80 2024-11-12T00:53:08,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/4706fb1493de4b6f9209fd8cae290282 is 50, key is test_row_0/C:col10/1731372786439/Put/seqid=0 2024-11-12T00:53:08,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372848275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:08,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741929_1105 (size=12301) 2024-11-12T00:53:08,280 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/4706fb1493de4b6f9209fd8cae290282 2024-11-12T00:53:08,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/f769e667204645c4acc9b98296956dd2 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f769e667204645c4acc9b98296956dd2 2024-11-12T00:53:08,291 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f769e667204645c4acc9b98296956dd2, entries=150, sequenceid=571, filesize=12.0 K 2024-11-12T00:53:08,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/7e183781a9c94aa6991c0f4dce7cec80 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/7e183781a9c94aa6991c0f4dce7cec80 2024-11-12T00:53:08,307 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/7e183781a9c94aa6991c0f4dce7cec80, entries=150, sequenceid=571, filesize=12.0 K 2024-11-12T00:53:08,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/4706fb1493de4b6f9209fd8cae290282 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/4706fb1493de4b6f9209fd8cae290282 2024-11-12T00:53:08,318 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/4706fb1493de4b6f9209fd8cae290282, entries=150, sequenceid=571, filesize=12.0 K 2024-11-12T00:53:08,321 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8f6f5fa6d51a99bb28036e719e157e24 in 899ms, sequenceid=571, compaction requested=false 2024-11-12T00:53:08,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:08,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:08,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-12T00:53:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-12T00:53:08,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-12T00:53:08,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5170 sec 2024-11-12T00:53:08,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.5220 sec 2024-11-12T00:53:08,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:08,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-12T00:53:08,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:08,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:08,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:08,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:08,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:08,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:08,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/3a045d5609f4489fb45d5a66d9019543 is 50, key is test_row_0/A:col10/1731372788785/Put/seqid=0 2024-11-12T00:53:08,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741930_1106 (size=12301) 2024-11-12T00:53:08,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372848822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:08,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372848935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:08,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:08,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:08,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T00:53:08,966 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:08,966 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-12T00:53:08,968 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T00:53:08,969 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:08,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T00:53:09,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T00:53:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:09,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372849145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:09,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/3a045d5609f4489fb45d5a66d9019543 2024-11-12T00:53:09,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/c4426fad1ad64cc38225f296eb3ba460 is 50, key is test_row_0/B:col10/1731372788785/Put/seqid=0 2024-11-12T00:53:09,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741931_1107 (size=12301) 2024-11-12T00:53:09,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T00:53:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:09,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T00:53:09,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T00:53:09,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:09,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:09,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:09,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372849456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:09,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T00:53:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:09,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T00:53:09,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/c4426fad1ad64cc38225f296eb3ba460 2024-11-12T00:53:09,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/ca3f84e21e4f4ebf952ca9de05d82ccb is 50, key is test_row_0/C:col10/1731372788785/Put/seqid=0 2024-11-12T00:53:09,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741932_1108 (size=12301) 2024-11-12T00:53:09,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T00:53:09,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:09,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:09,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:09,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T00:53:09,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:09,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:09,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:09,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:09,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372849976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:10,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/ca3f84e21e4f4ebf952ca9de05d82ccb 2024-11-12T00:53:10,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/3a045d5609f4489fb45d5a66d9019543 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3a045d5609f4489fb45d5a66d9019543 2024-11-12T00:53:10,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3a045d5609f4489fb45d5a66d9019543, entries=150, sequenceid=592, filesize=12.0 K 2024-11-12T00:53:10,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/c4426fad1ad64cc38225f296eb3ba460 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/c4426fad1ad64cc38225f296eb3ba460 2024-11-12T00:53:10,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T00:53:10,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:10,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:10,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:10,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/c4426fad1ad64cc38225f296eb3ba460, entries=150, sequenceid=592, filesize=12.0 K 2024-11-12T00:53:10,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:10,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:10,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/ca3f84e21e4f4ebf952ca9de05d82ccb as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ca3f84e21e4f4ebf952ca9de05d82ccb 2024-11-12T00:53:10,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ca3f84e21e4f4ebf952ca9de05d82ccb, entries=150, sequenceid=592, filesize=12.0 K 2024-11-12T00:53:10,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 8f6f5fa6d51a99bb28036e719e157e24 in 1259ms, sequenceid=592, compaction requested=true 2024-11-12T00:53:10,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:10,047 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:10,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:53:10,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:10,048 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:10,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:53:10,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:10,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:53:10,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:10,049 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:10,050 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/A is initiating minor compaction (all files) 2024-11-12T00:53:10,050 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/A in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:10,050 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:10,050 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/B is initiating minor compaction (all files) 2024-11-12T00:53:10,050 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/12e4a441d30d49c7825953e1b5184758, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f769e667204645c4acc9b98296956dd2, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3a045d5609f4489fb45d5a66d9019543] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=37.0 K 2024-11-12T00:53:10,050 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/B in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:10,050 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e067a6ecb224dee97913d98f5b4b49f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/7e183781a9c94aa6991c0f4dce7cec80, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/c4426fad1ad64cc38225f296eb3ba460] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=37.0 K 2024-11-12T00:53:10,050 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12e4a441d30d49c7825953e1b5184758, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1731372785767 2024-11-12T00:53:10,051 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting f769e667204645c4acc9b98296956dd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731372786418 2024-11-12T00:53:10,051 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e067a6ecb224dee97913d98f5b4b49f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1731372785767 2024-11-12T00:53:10,051 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a045d5609f4489fb45d5a66d9019543, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1731372787635 2024-11-12T00:53:10,051 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e183781a9c94aa6991c0f4dce7cec80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731372786418 2024-11-12T00:53:10,051 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting c4426fad1ad64cc38225f296eb3ba460, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1731372787635 2024-11-12T00:53:10,058 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:10,059 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3a5110ec1b5347889076cb5cbfc6aca9 is 50, key is test_row_0/B:col10/1731372788785/Put/seqid=0 2024-11-12T00:53:10,061 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#A#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:10,061 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/abe2fa59e924464c8fb0a9958b9a8028 is 50, key is test_row_0/A:col10/1731372788785/Put/seqid=0 2024-11-12T00:53:10,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741933_1109 (size=13391) 2024-11-12T00:53:10,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741934_1110 (size=13391) 2024-11-12T00:53:10,081 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/3a5110ec1b5347889076cb5cbfc6aca9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3a5110ec1b5347889076cb5cbfc6aca9 2024-11-12T00:53:10,088 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/B of 8f6f5fa6d51a99bb28036e719e157e24 into 3a5110ec1b5347889076cb5cbfc6aca9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:10,088 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:10,088 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/B, priority=13, startTime=1731372790047; duration=0sec 2024-11-12T00:53:10,088 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:10,089 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:B 2024-11-12T00:53:10,090 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:10,091 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:10,091 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/C is initiating minor compaction (all files) 2024-11-12T00:53:10,091 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/C in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:10,091 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/0697dde878d54eecb341a235c277ea34, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/4706fb1493de4b6f9209fd8cae290282, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ca3f84e21e4f4ebf952ca9de05d82ccb] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=37.0 K 2024-11-12T00:53:10,092 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 0697dde878d54eecb341a235c277ea34, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1731372785767 2024-11-12T00:53:10,092 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 4706fb1493de4b6f9209fd8cae290282, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731372786418 2024-11-12T00:53:10,092 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting ca3f84e21e4f4ebf952ca9de05d82ccb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1731372787635 2024-11-12T00:53:10,100 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:10,101 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/7e9dbfbcb78c4e8fad1aba205bdbb2d1 is 50, key is test_row_0/C:col10/1731372788785/Put/seqid=0 2024-11-12T00:53:10,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T00:53:10,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741935_1111 (size=13391) 2024-11-12T00:53:10,112 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/7e9dbfbcb78c4e8fad1aba205bdbb2d1 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/7e9dbfbcb78c4e8fad1aba205bdbb2d1 2024-11-12T00:53:10,118 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/C of 8f6f5fa6d51a99bb28036e719e157e24 into 7e9dbfbcb78c4e8fad1aba205bdbb2d1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:10,118 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:10,118 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/C, priority=13, startTime=1731372790048; duration=0sec 2024-11-12T00:53:10,118 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:10,118 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:C 2024-11-12T00:53:10,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T00:53:10,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:10,191 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T00:53:10,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:10,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:10,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:10,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:10,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:10,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:10,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/71356b8410ba402784969b835ca35c76 is 50, key is test_row_0/A:col10/1731372788821/Put/seqid=0 2024-11-12T00:53:10,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741936_1112 (size=12301) 2024-11-12T00:53:10,476 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/abe2fa59e924464c8fb0a9958b9a8028 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/abe2fa59e924464c8fb0a9958b9a8028 2024-11-12T00:53:10,482 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/A of 8f6f5fa6d51a99bb28036e719e157e24 into abe2fa59e924464c8fb0a9958b9a8028(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:10,482 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:10,482 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/A, priority=13, startTime=1731372790047; duration=0sec 2024-11-12T00:53:10,482 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:10,482 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:A 2024-11-12T00:53:10,600 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=605 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/71356b8410ba402784969b835ca35c76 2024-11-12T00:53:10,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/89ef8f3830e14e3bb6c194e63941620b is 50, key is test_row_0/B:col10/1731372788821/Put/seqid=0 2024-11-12T00:53:10,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741937_1113 (size=12301) 2024-11-12T00:53:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:10,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. as already flushing 2024-11-12T00:53:10,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372850887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:10,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731372790858, max timestamp=1731372790887Num uniques 859; 2024-11-12T00:53:10,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731372790858, max timestamp=1731372790887Num uniques 859; 2024-11-12T00:53:10,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731372790858, max timestamp=1731372790887Num uniques 859; 2024-11-12T00:53:10,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:53:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:10,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:53:10,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:53:10,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372850979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:10,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:10,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:10,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:10,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:10,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372850984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:10,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:10,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:10,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:10,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45780 deadline: 1731372850986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:10,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:10,991 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 8f6f5fa6d51a99bb28036e719e157e24 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:10,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:10,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:10,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:11,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372851006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:11,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:11,016 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=605 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/89ef8f3830e14e3bb6c194e63941620b 2024-11-12T00:53:11,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/c151a6c83a404af39ffca6a5fecd55bf is 50, key is test_row_0/C:col10/1731372788821/Put/seqid=0 2024-11-12T00:53:11,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741938_1114 (size=12301) 2024-11-12T00:53:11,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:11,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372851096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372851096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T00:53:11,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372851215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:11,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372851316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:11,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372851316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:11,432 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=605 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/c151a6c83a404af39ffca6a5fecd55bf 2024-11-12T00:53:11,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/71356b8410ba402784969b835ca35c76 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/71356b8410ba402784969b835ca35c76 2024-11-12T00:53:11,442 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/71356b8410ba402784969b835ca35c76, entries=150, sequenceid=605, filesize=12.0 K 2024-11-12T00:53:11,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/89ef8f3830e14e3bb6c194e63941620b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/89ef8f3830e14e3bb6c194e63941620b 2024-11-12T00:53:11,448 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/89ef8f3830e14e3bb6c194e63941620b, entries=150, sequenceid=605, filesize=12.0 K 2024-11-12T00:53:11,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/c151a6c83a404af39ffca6a5fecd55bf as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/c151a6c83a404af39ffca6a5fecd55bf 2024-11-12T00:53:11,454 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/c151a6c83a404af39ffca6a5fecd55bf, entries=150, sequenceid=605, filesize=12.0 K 2024-11-12T00:53:11,455 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8f6f5fa6d51a99bb28036e719e157e24 in 1264ms, sequenceid=605, compaction requested=false 2024-11-12T00:53:11,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:11,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:11,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-12T00:53:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-12T00:53:11,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-12T00:53:11,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4880 sec 2024-11-12T00:53:11,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.4920 sec 2024-11-12T00:53:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:11,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-12T00:53:11,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:11,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:11,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:11,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:11,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:11,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:11,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/3ccd4ac2931d4089a4f2f644cc65fb59 is 50, key is test_row_0/A:col10/1731372791529/Put/seqid=0 2024-11-12T00:53:11,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741939_1115 (size=14741) 2024-11-12T00:53:11,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/3ccd4ac2931d4089a4f2f644cc65fb59 2024-11-12T00:53:11,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/7070fab21db84fcd96a98dc5044e227f is 50, key is test_row_0/B:col10/1731372791529/Put/seqid=0 2024-11-12T00:53:11,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741940_1116 (size=12301) 2024-11-12T00:53:11,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372851565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:11,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372851626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:11,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372851626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:11,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372851675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:11,696 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-12T00:53:11,696 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:11,699 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-12T00:53:11,699 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:11,701 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:11,701 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:11,701 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-12T00:53:11,701 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:11,701 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:11,701 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:11,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:11,702 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:11,702 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:11,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:11,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:11,704 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-12T00:53:11,704 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:11,704 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:11,704 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:11,704 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:11,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372851886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:11,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:11,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/7070fab21db84fcd96a98dc5044e227f 2024-11-12T00:53:11,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/dc52031f68374dee973b3365276af977 is 50, key is test_row_0/C:col10/1731372791529/Put/seqid=0 2024-11-12T00:53:11,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741941_1117 (size=12301) 2024-11-12T00:53:12,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:12,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1731372852136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45764 deadline: 1731372852136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:12,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:12,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1731372852196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:12,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/dc52031f68374dee973b3365276af977 2024-11-12T00:53:12,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/3ccd4ac2931d4089a4f2f644cc65fb59 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3ccd4ac2931d4089a4f2f644cc65fb59 2024-11-12T00:53:12,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3ccd4ac2931d4089a4f2f644cc65fb59, entries=200, sequenceid=642, filesize=14.4 K 2024-11-12T00:53:12,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/7070fab21db84fcd96a98dc5044e227f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/7070fab21db84fcd96a98dc5044e227f 2024-11-12T00:53:12,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/7070fab21db84fcd96a98dc5044e227f, entries=150, sequenceid=642, filesize=12.0 K 2024-11-12T00:53:12,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/dc52031f68374dee973b3365276af977 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/dc52031f68374dee973b3365276af977 2024-11-12T00:53:12,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/dc52031f68374dee973b3365276af977, entries=150, sequenceid=642, filesize=12.0 K 2024-11-12T00:53:12,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 8f6f5fa6d51a99bb28036e719e157e24 in 883ms, sequenceid=642, compaction requested=true 2024-11-12T00:53:12,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:12,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:53:12,415 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:12,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:12,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:53:12,415 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:12,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:12,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f6f5fa6d51a99bb28036e719e157e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:53:12,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:12,416 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:12,416 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:12,416 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/B is initiating minor compaction (all files) 2024-11-12T00:53:12,416 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/A is initiating minor compaction (all files) 2024-11-12T00:53:12,416 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/B in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:12,416 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/A in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:12,416 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/abe2fa59e924464c8fb0a9958b9a8028, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/71356b8410ba402784969b835ca35c76, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3ccd4ac2931d4089a4f2f644cc65fb59] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=39.5 K 2024-11-12T00:53:12,416 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3a5110ec1b5347889076cb5cbfc6aca9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/89ef8f3830e14e3bb6c194e63941620b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/7070fab21db84fcd96a98dc5044e227f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=37.1 K 2024-11-12T00:53:12,417 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting abe2fa59e924464c8fb0a9958b9a8028, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1731372787635 2024-11-12T00:53:12,417 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a5110ec1b5347889076cb5cbfc6aca9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1731372787635 2024-11-12T00:53:12,417 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 71356b8410ba402784969b835ca35c76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731372788796 2024-11-12T00:53:12,417 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89ef8f3830e14e3bb6c194e63941620b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731372788796 2024-11-12T00:53:12,417 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ccd4ac2931d4089a4f2f644cc65fb59, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=642, earliestPutTs=1731372790969 2024-11-12T00:53:12,417 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7070fab21db84fcd96a98dc5044e227f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=642, earliestPutTs=1731372790969 2024-11-12T00:53:12,426 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#B#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:12,426 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#A#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:12,426 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/281a6195c7234c4da15579bb2e6275c4 is 50, key is test_row_0/B:col10/1731372791529/Put/seqid=0 2024-11-12T00:53:12,426 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/fe6746cd1396441baef926e7dd448a73 is 50, key is test_row_0/A:col10/1731372791529/Put/seqid=0 2024-11-12T00:53:12,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741943_1119 (size=13493) 2024-11-12T00:53:12,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741942_1118 (size=13493) 2024-11-12T00:53:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:12,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T00:53:12,711 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-12T00:53:12,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:12,711 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:12,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:12,711 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:12,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:12,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:12,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:12,711 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:12,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:12,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:12,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/e971eb1732224a4c898fc92a1b1a0ba7 is 50, key is test_row_0/A:col10/1731372792706/Put/seqid=0 2024-11-12T00:53:12,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741944_1120 (size=12301) 2024-11-12T00:53:12,844 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/281a6195c7234c4da15579bb2e6275c4 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/281a6195c7234c4da15579bb2e6275c4 2024-11-12T00:53:12,845 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/fe6746cd1396441baef926e7dd448a73 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/fe6746cd1396441baef926e7dd448a73 2024-11-12T00:53:12,851 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/B of 8f6f5fa6d51a99bb28036e719e157e24 into 281a6195c7234c4da15579bb2e6275c4(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:12,851 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:12,851 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/B, priority=13, startTime=1731372792415; duration=0sec 2024-11-12T00:53:12,851 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/A of 8f6f5fa6d51a99bb28036e719e157e24 into fe6746cd1396441baef926e7dd448a73(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:12,851 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:12,851 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/A, priority=13, startTime=1731372792414; duration=0sec 2024-11-12T00:53:12,851 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:12,851 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:B 2024-11-12T00:53:12,851 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:12,851 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:A 2024-11-12T00:53:12,851 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:12,852 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:12,852 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 8f6f5fa6d51a99bb28036e719e157e24/C is initiating minor compaction (all files) 2024-11-12T00:53:12,852 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f6f5fa6d51a99bb28036e719e157e24/C in TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:12,853 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/7e9dbfbcb78c4e8fad1aba205bdbb2d1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/c151a6c83a404af39ffca6a5fecd55bf, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/dc52031f68374dee973b3365276af977] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp, totalSize=37.1 K 2024-11-12T00:53:12,853 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e9dbfbcb78c4e8fad1aba205bdbb2d1, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1731372787635 2024-11-12T00:53:12,853 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting c151a6c83a404af39ffca6a5fecd55bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731372788796 2024-11-12T00:53:12,854 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc52031f68374dee973b3365276af977, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=642, earliestPutTs=1731372790969 2024-11-12T00:53:12,863 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f6f5fa6d51a99bb28036e719e157e24#C#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:12,863 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/decd936d75f2430aacb1bee8af3d0921 is 50, key is test_row_0/C:col10/1731372791529/Put/seqid=0 2024-11-12T00:53:12,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741945_1121 (size=13493) 2024-11-12T00:53:13,010 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-12T00:53:13,010 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:13,011 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:13,011 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:13,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:13,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/e971eb1732224a4c898fc92a1b1a0ba7 2024-11-12T00:53:13,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T00:53:13,136 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:13,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/74d4d704fb594a31b0acc00d05e0e48e is 50, key is test_row_0/B:col10/1731372792706/Put/seqid=0 2024-11-12T00:53:13,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741946_1122 (size=12301) 2024-11-12T00:53:13,157 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-12T00:53:13,157 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-12T00:53:13,157 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:13,157 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:13,157 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:13,157 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:13,157 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:13,157 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:13,157 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:13,157 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:13,282 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/decd936d75f2430aacb1bee8af3d0921 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/decd936d75f2430aacb1bee8af3d0921 2024-11-12T00:53:13,288 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f6f5fa6d51a99bb28036e719e157e24/C of 8f6f5fa6d51a99bb28036e719e157e24 into decd936d75f2430aacb1bee8af3d0921(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:13,288 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:13,288 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., storeName=8f6f5fa6d51a99bb28036e719e157e24/C, priority=13, startTime=1731372792415; duration=0sec 2024-11-12T00:53:13,288 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:13,288 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f6f5fa6d51a99bb28036e719e157e24:C 2024-11-12T00:53:13,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/74d4d704fb594a31b0acc00d05e0e48e 2024-11-12T00:53:13,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/2dbe29f4c114465f9984ca6d75305883 is 50, key is test_row_0/C:col10/1731372792706/Put/seqid=0 2024-11-12T00:53:13,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741947_1123 (size=12301) 2024-11-12T00:53:13,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/2dbe29f4c114465f9984ca6d75305883 2024-11-12T00:53:13,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/e971eb1732224a4c898fc92a1b1a0ba7 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/e971eb1732224a4c898fc92a1b1a0ba7 2024-11-12T00:53:13,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/e971eb1732224a4c898fc92a1b1a0ba7, entries=150, sequenceid=653, filesize=12.0 K 2024-11-12T00:53:13,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/74d4d704fb594a31b0acc00d05e0e48e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/74d4d704fb594a31b0acc00d05e0e48e 2024-11-12T00:53:13,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/74d4d704fb594a31b0acc00d05e0e48e, entries=150, sequenceid=653, filesize=12.0 K 2024-11-12T00:53:13,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/2dbe29f4c114465f9984ca6d75305883 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/2dbe29f4c114465f9984ca6d75305883 2024-11-12T00:53:13,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/2dbe29f4c114465f9984ca6d75305883, entries=150, sequenceid=653, filesize=12.0 K 2024-11-12T00:53:13,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=20.13 KB/20610 for 8f6f5fa6d51a99bb28036e719e157e24 in 1274ms, sequenceid=653, compaction requested=false 2024-11-12T00:53:13,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f6f5fa6d51a99bb28036e719e157e24: 2024-11-12T00:53:14,053 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T00:53:14,857 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-12T00:53:14,858 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:14,858 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:14,858 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:14,858 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:14,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T00:53:14,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-12T00:53:14,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-12T00:53:14,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-11-12T00:53:14,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 147 2024-11-12T00:53:14,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-12T00:53:14,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T00:53:14,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2743 2024-11-12T00:53:14,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2762 2024-11-12T00:53:14,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T00:53:14,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1219 2024-11-12T00:53:14,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3657 rows 2024-11-12T00:53:14,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1215 2024-11-12T00:53:14,859 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3643 rows 2024-11-12T00:53:14,859 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T00:53:14,859 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T00:53:14,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:14,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:14,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:14,864 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T00:53:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T00:53:14,874 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372794874"}]},"ts":"1731372794874"} 2024-11-12T00:53:14,877 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T00:53:14,877 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T00:53:14,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T00:53:14,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f6f5fa6d51a99bb28036e719e157e24, UNASSIGN}] 2024-11-12T00:53:14,887 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f6f5fa6d51a99bb28036e719e157e24, UNASSIGN 2024-11-12T00:53:14,888 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=8f6f5fa6d51a99bb28036e719e157e24, regionState=CLOSING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:14,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f6f5fa6d51a99bb28036e719e157e24, UNASSIGN because future has completed 2024-11-12T00:53:14,893 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T00:53:14,893 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:53:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T00:53:15,051 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(122): Close 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:15,051 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T00:53:15,052 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1722): Closing 8f6f5fa6d51a99bb28036e719e157e24, disabling compactions & flushes 2024-11-12T00:53:15,052 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:15,052 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:15,052 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. after waiting 0 ms 2024-11-12T00:53:15,052 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:15,052 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(2902): Flushing 8f6f5fa6d51a99bb28036e719e157e24 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T00:53:15,052 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=A 2024-11-12T00:53:15,053 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:15,053 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=B 2024-11-12T00:53:15,053 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:15,053 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8f6f5fa6d51a99bb28036e719e157e24, store=C 2024-11-12T00:53:15,053 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:15,057 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/08b8dda2f5ae41299c0dbb1871f3ccb0 is 50, key is test_row_0/A:col10/1731372794856/Put/seqid=0 2024-11-12T00:53:15,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741948_1124 (size=12301) 2024-11-12T00:53:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T00:53:15,464 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=663 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/08b8dda2f5ae41299c0dbb1871f3ccb0 2024-11-12T00:53:15,478 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/80c2316dfd7a490c9d5e1c362ae4eb8e is 50, key is test_row_0/B:col10/1731372794856/Put/seqid=0 2024-11-12T00:53:15,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741949_1125 (size=12301) 2024-11-12T00:53:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T00:53:15,890 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=663 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/80c2316dfd7a490c9d5e1c362ae4eb8e 2024-11-12T00:53:15,900 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/1fc7afedfb334288beb59b0712f47bf2 is 50, key is test_row_0/C:col10/1731372794856/Put/seqid=0 2024-11-12T00:53:15,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741950_1126 (size=12301) 2024-11-12T00:53:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T00:53:16,305 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=663 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/1fc7afedfb334288beb59b0712f47bf2 2024-11-12T00:53:16,311 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/A/08b8dda2f5ae41299c0dbb1871f3ccb0 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/08b8dda2f5ae41299c0dbb1871f3ccb0 2024-11-12T00:53:16,317 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/08b8dda2f5ae41299c0dbb1871f3ccb0, entries=150, sequenceid=663, filesize=12.0 K 2024-11-12T00:53:16,318 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/B/80c2316dfd7a490c9d5e1c362ae4eb8e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/80c2316dfd7a490c9d5e1c362ae4eb8e 2024-11-12T00:53:16,325 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/80c2316dfd7a490c9d5e1c362ae4eb8e, entries=150, sequenceid=663, filesize=12.0 K 2024-11-12T00:53:16,327 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/.tmp/C/1fc7afedfb334288beb59b0712f47bf2 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/1fc7afedfb334288beb59b0712f47bf2 2024-11-12T00:53:16,334 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/1fc7afedfb334288beb59b0712f47bf2, entries=150, sequenceid=663, filesize=12.0 K 2024-11-12T00:53:16,335 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8f6f5fa6d51a99bb28036e719e157e24 in 1283ms, sequenceid=663, compaction requested=true 2024-11-12T00:53:16,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/82e45c82574c44b6b5ca96ec3fff8fcf, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/922d8e82128d42c0a9e4e5f8600a4f63, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/69876bcf13b04d1b979309fc2d9ba1ea, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/a3bd343a5d2949a8b3c7fbdb2690e3b9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/ed4ca10f71ab4b1ba67ce0f6d1d47871, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/73592ef238444a6aa034b8725d2b21ba, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/815db4ddd4f24d6bbb6ce59087034acb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3b8b967e6582488fabe5c3cc91df2a58, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/14c7e4de5b99451698e5144b451a8a7b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/56735bb1fd804a8f93ca0c97778b5fb1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/24c538ec2b00491894771b75fc8ff455, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3fcd15243c124603883aa882bb189ffb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/a423238bb7ba4886b4c55162cbd26954, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/9c9c498cc4084479a4a1fb17d456806d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/ce2b925266d04a248d63fb0377bc02cc, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/c0b90393ea1f4a768c17aa72647d9d5f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/513e218c2c844ea596a968e6a497fab3, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f147feb204004f139530262d945fa9d9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/7db5e14c43d444b4816ecb9e07984fc2, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/80fff96ae6924dbe81af67955b725fef, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/63744c4a12904953b75c6fb7251dc13b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/7b27e09da08d4902aef3504639ade6e8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/6d8a8fb5e8ac4615a20c6de15a9d3ea1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f9adc5fe556042bd9d52e3896ff710ae, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/4d8fa101cf8c4c71a87ef695bb3bf4a3, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/5a7f318da2a94cf5a7f6c757fbca2902, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/99e2bff1a39042f2938eafe868ad4150, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/9403c78408a246aab911e628505de394, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/883f6d4a17aa45d391fe1705778b5e33, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/12e4a441d30d49c7825953e1b5184758, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f769e667204645c4acc9b98296956dd2, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/abe2fa59e924464c8fb0a9958b9a8028, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3a045d5609f4489fb45d5a66d9019543, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/71356b8410ba402784969b835ca35c76, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3ccd4ac2931d4089a4f2f644cc65fb59] to archive 2024-11-12T00:53:16,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:53:16,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/82e45c82574c44b6b5ca96ec3fff8fcf to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/82e45c82574c44b6b5ca96ec3fff8fcf 2024-11-12T00:53:16,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/922d8e82128d42c0a9e4e5f8600a4f63 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/922d8e82128d42c0a9e4e5f8600a4f63 2024-11-12T00:53:16,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/69876bcf13b04d1b979309fc2d9ba1ea to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/69876bcf13b04d1b979309fc2d9ba1ea 2024-11-12T00:53:16,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/a3bd343a5d2949a8b3c7fbdb2690e3b9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/a3bd343a5d2949a8b3c7fbdb2690e3b9 2024-11-12T00:53:16,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/ed4ca10f71ab4b1ba67ce0f6d1d47871 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/ed4ca10f71ab4b1ba67ce0f6d1d47871 2024-11-12T00:53:16,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/73592ef238444a6aa034b8725d2b21ba to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/73592ef238444a6aa034b8725d2b21ba 2024-11-12T00:53:16,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/815db4ddd4f24d6bbb6ce59087034acb to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/815db4ddd4f24d6bbb6ce59087034acb 2024-11-12T00:53:16,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3b8b967e6582488fabe5c3cc91df2a58 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3b8b967e6582488fabe5c3cc91df2a58 2024-11-12T00:53:16,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/14c7e4de5b99451698e5144b451a8a7b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/14c7e4de5b99451698e5144b451a8a7b 2024-11-12T00:53:16,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/56735bb1fd804a8f93ca0c97778b5fb1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/56735bb1fd804a8f93ca0c97778b5fb1 2024-11-12T00:53:16,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/24c538ec2b00491894771b75fc8ff455 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/24c538ec2b00491894771b75fc8ff455 2024-11-12T00:53:16,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3fcd15243c124603883aa882bb189ffb to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3fcd15243c124603883aa882bb189ffb 2024-11-12T00:53:16,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/a423238bb7ba4886b4c55162cbd26954 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/a423238bb7ba4886b4c55162cbd26954 2024-11-12T00:53:16,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/9c9c498cc4084479a4a1fb17d456806d to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/9c9c498cc4084479a4a1fb17d456806d 2024-11-12T00:53:16,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/ce2b925266d04a248d63fb0377bc02cc to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/ce2b925266d04a248d63fb0377bc02cc 2024-11-12T00:53:16,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/c0b90393ea1f4a768c17aa72647d9d5f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/c0b90393ea1f4a768c17aa72647d9d5f 2024-11-12T00:53:16,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/513e218c2c844ea596a968e6a497fab3 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/513e218c2c844ea596a968e6a497fab3 2024-11-12T00:53:16,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f147feb204004f139530262d945fa9d9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f147feb204004f139530262d945fa9d9 2024-11-12T00:53:16,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/7db5e14c43d444b4816ecb9e07984fc2 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/7db5e14c43d444b4816ecb9e07984fc2 2024-11-12T00:53:16,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/80fff96ae6924dbe81af67955b725fef to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/80fff96ae6924dbe81af67955b725fef 2024-11-12T00:53:16,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/63744c4a12904953b75c6fb7251dc13b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/63744c4a12904953b75c6fb7251dc13b 2024-11-12T00:53:16,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/7b27e09da08d4902aef3504639ade6e8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/7b27e09da08d4902aef3504639ade6e8 2024-11-12T00:53:16,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/6d8a8fb5e8ac4615a20c6de15a9d3ea1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/6d8a8fb5e8ac4615a20c6de15a9d3ea1 2024-11-12T00:53:16,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f9adc5fe556042bd9d52e3896ff710ae to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f9adc5fe556042bd9d52e3896ff710ae 2024-11-12T00:53:16,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/4d8fa101cf8c4c71a87ef695bb3bf4a3 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/4d8fa101cf8c4c71a87ef695bb3bf4a3 2024-11-12T00:53:16,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/5a7f318da2a94cf5a7f6c757fbca2902 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/5a7f318da2a94cf5a7f6c757fbca2902 2024-11-12T00:53:16,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/99e2bff1a39042f2938eafe868ad4150 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/99e2bff1a39042f2938eafe868ad4150 2024-11-12T00:53:16,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/9403c78408a246aab911e628505de394 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/9403c78408a246aab911e628505de394 2024-11-12T00:53:16,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/883f6d4a17aa45d391fe1705778b5e33 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/883f6d4a17aa45d391fe1705778b5e33 2024-11-12T00:53:16,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/12e4a441d30d49c7825953e1b5184758 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/12e4a441d30d49c7825953e1b5184758 2024-11-12T00:53:16,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f769e667204645c4acc9b98296956dd2 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/f769e667204645c4acc9b98296956dd2 2024-11-12T00:53:16,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/abe2fa59e924464c8fb0a9958b9a8028 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/abe2fa59e924464c8fb0a9958b9a8028 2024-11-12T00:53:16,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3a045d5609f4489fb45d5a66d9019543 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3a045d5609f4489fb45d5a66d9019543 2024-11-12T00:53:16,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/71356b8410ba402784969b835ca35c76 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/71356b8410ba402784969b835ca35c76 2024-11-12T00:53:16,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3ccd4ac2931d4089a4f2f644cc65fb59 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/3ccd4ac2931d4089a4f2f644cc65fb59 2024-11-12T00:53:16,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/710a69c7698a433d890c941434dc6342, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/9c0f15d895454b8a91565d56266f1629, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/6bddb507666d4fb5b6987477f08efecb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/770214db633845a7ac41e1ebd2b2af73, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/060c8c762e0544f9b526d744d4210a98, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/993e670d99344685b52696ab27b8a958, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/be4bbbc18571437e872ad2970e24262b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e17bac0f06a464cb168ef9490a2cf15, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/479bc5c78b2542619d5c699418935363, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e5607d3e66c4a54a371add01f840a48, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1751959ecadf419bbe5640295c8b60cc, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/8ca8536fb08e48f3a46a112d0a73c90f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/5f8a547b0a2446f485358578b5ebb277, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3782a7daae9847fd8ce7885be1304b72, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4ef7bd8b70804f379ae33280392e735e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/6745d25257a640b69de7552ae95e9feb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4791309658d4440f8721d93b8258d60d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/423491a5362b415f9f10f28762937467, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/bfb677e7db6c46b29db9b099b4fbace8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/313358ba9bd7491cb1e1f48776a0fe93, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/d89c7cac4ed740d9b932f74e302a2ed6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3022c7a4f4ef4160b08600cc888df906, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1c2ee1ee330a4d839d700cced0fdbc12, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4cf91fcd480c4243ab0da9900b86c08c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/98a88e1f90bf46789ad26986054a92bf, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/9f2d79e96d6f49aa926f4f5f7fdd13ef, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/c963e00be1e34d72822b7ba27d673445, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1d5d60ed833a4fa0a1a9e77a51cf495b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e067a6ecb224dee97913d98f5b4b49f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/50d208304d224c8f856f79e3657f78d5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/7e183781a9c94aa6991c0f4dce7cec80, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3a5110ec1b5347889076cb5cbfc6aca9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/c4426fad1ad64cc38225f296eb3ba460, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/89ef8f3830e14e3bb6c194e63941620b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/7070fab21db84fcd96a98dc5044e227f] to archive 2024-11-12T00:53:16,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:53:16,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/710a69c7698a433d890c941434dc6342 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/710a69c7698a433d890c941434dc6342 2024-11-12T00:53:16,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/9c0f15d895454b8a91565d56266f1629 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/9c0f15d895454b8a91565d56266f1629 2024-11-12T00:53:16,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/6bddb507666d4fb5b6987477f08efecb to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/6bddb507666d4fb5b6987477f08efecb 2024-11-12T00:53:16,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/770214db633845a7ac41e1ebd2b2af73 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/770214db633845a7ac41e1ebd2b2af73 2024-11-12T00:53:16,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/060c8c762e0544f9b526d744d4210a98 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/060c8c762e0544f9b526d744d4210a98 2024-11-12T00:53:16,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/993e670d99344685b52696ab27b8a958 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/993e670d99344685b52696ab27b8a958 2024-11-12T00:53:16,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/be4bbbc18571437e872ad2970e24262b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/be4bbbc18571437e872ad2970e24262b 2024-11-12T00:53:16,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e17bac0f06a464cb168ef9490a2cf15 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e17bac0f06a464cb168ef9490a2cf15 2024-11-12T00:53:16,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/479bc5c78b2542619d5c699418935363 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/479bc5c78b2542619d5c699418935363 2024-11-12T00:53:16,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e5607d3e66c4a54a371add01f840a48 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e5607d3e66c4a54a371add01f840a48 2024-11-12T00:53:16,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1751959ecadf419bbe5640295c8b60cc to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1751959ecadf419bbe5640295c8b60cc 2024-11-12T00:53:16,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/8ca8536fb08e48f3a46a112d0a73c90f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/8ca8536fb08e48f3a46a112d0a73c90f 2024-11-12T00:53:16,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/5f8a547b0a2446f485358578b5ebb277 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/5f8a547b0a2446f485358578b5ebb277 2024-11-12T00:53:16,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3782a7daae9847fd8ce7885be1304b72 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3782a7daae9847fd8ce7885be1304b72 2024-11-12T00:53:16,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4ef7bd8b70804f379ae33280392e735e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4ef7bd8b70804f379ae33280392e735e 2024-11-12T00:53:16,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/6745d25257a640b69de7552ae95e9feb to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/6745d25257a640b69de7552ae95e9feb 2024-11-12T00:53:16,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4791309658d4440f8721d93b8258d60d to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4791309658d4440f8721d93b8258d60d 2024-11-12T00:53:16,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/423491a5362b415f9f10f28762937467 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/423491a5362b415f9f10f28762937467 2024-11-12T00:53:16,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/bfb677e7db6c46b29db9b099b4fbace8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/bfb677e7db6c46b29db9b099b4fbace8 2024-11-12T00:53:16,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/313358ba9bd7491cb1e1f48776a0fe93 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/313358ba9bd7491cb1e1f48776a0fe93 2024-11-12T00:53:16,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/d89c7cac4ed740d9b932f74e302a2ed6 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/d89c7cac4ed740d9b932f74e302a2ed6 2024-11-12T00:53:16,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3022c7a4f4ef4160b08600cc888df906 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3022c7a4f4ef4160b08600cc888df906 2024-11-12T00:53:16,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1c2ee1ee330a4d839d700cced0fdbc12 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1c2ee1ee330a4d839d700cced0fdbc12 2024-11-12T00:53:16,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4cf91fcd480c4243ab0da9900b86c08c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/4cf91fcd480c4243ab0da9900b86c08c 2024-11-12T00:53:16,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/98a88e1f90bf46789ad26986054a92bf to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/98a88e1f90bf46789ad26986054a92bf 2024-11-12T00:53:16,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/9f2d79e96d6f49aa926f4f5f7fdd13ef to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/9f2d79e96d6f49aa926f4f5f7fdd13ef 2024-11-12T00:53:16,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/c963e00be1e34d72822b7ba27d673445 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/c963e00be1e34d72822b7ba27d673445 2024-11-12T00:53:16,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1d5d60ed833a4fa0a1a9e77a51cf495b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/1d5d60ed833a4fa0a1a9e77a51cf495b 2024-11-12T00:53:16,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e067a6ecb224dee97913d98f5b4b49f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3e067a6ecb224dee97913d98f5b4b49f 2024-11-12T00:53:16,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/50d208304d224c8f856f79e3657f78d5 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/50d208304d224c8f856f79e3657f78d5 2024-11-12T00:53:16,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/7e183781a9c94aa6991c0f4dce7cec80 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/7e183781a9c94aa6991c0f4dce7cec80 2024-11-12T00:53:16,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3a5110ec1b5347889076cb5cbfc6aca9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/3a5110ec1b5347889076cb5cbfc6aca9 2024-11-12T00:53:16,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/c4426fad1ad64cc38225f296eb3ba460 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/c4426fad1ad64cc38225f296eb3ba460 2024-11-12T00:53:16,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/89ef8f3830e14e3bb6c194e63941620b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/89ef8f3830e14e3bb6c194e63941620b 2024-11-12T00:53:16,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/7070fab21db84fcd96a98dc5044e227f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/7070fab21db84fcd96a98dc5044e227f 2024-11-12T00:53:16,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/82beca1bdb3c4e28998cf9a32529f9d2, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/7cf39364d2fb4299a603109a18b10ff9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/c2ba89d85fee4d52be3497418998d020, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/590740818d9f4f42a168a8e8c637117c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/792347adf7ab466eadd05e62e0f03c69, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/1735cba328b3442b808e4662791d624c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9d711b159c28423a97b4666606165356, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/de8029b0b60f440da0cad50b65fd6b21, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/764e270e3a9847b8806bffdf4bcb27de, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/6da97efcfcbc40e7be4917cb20228b32, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/4b9ce6f3b9a5457fa2df5a668518d29f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/11765aa085ce4491bc4aefd93f618bae, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/44192a7f65794bdb86dcb7e54fae54b2, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/bf21d57b9c77488e92d9d163abab33aa, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/b3470746cae64bb6a128f2f996dc2ba6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/6e9bfc64a77b43fa9fc2560ba15e0c9e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9ff6a8b03fd34e579887890c3eee54b9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/46aeea6997684ce5887bbd0e5320aedc, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/5a63fe0006034bcbbd140a14f0d01259, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ecb3a955add048fd9c0c4dc401cd1a5f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9ae217f47fe9456a8d0a10656657b822, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/f76850fbf3ce40a997ecad47b970f357, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/e526ac5850924f749031294a5fe836bc, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ce52b7cdd38c486e856935d41474936b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/32a09dd2da614faa82e45e244466c770, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/3214525920794454ae99ffa28dfdc5ce, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/0ea98005f161445eafe643bfdb51066f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/82bc1a6387ac408b9e75b833e0e53b84, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/0697dde878d54eecb341a235c277ea34, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/bd8a6eaa594a44eba6f76ff754dc887f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/4706fb1493de4b6f9209fd8cae290282, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/7e9dbfbcb78c4e8fad1aba205bdbb2d1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ca3f84e21e4f4ebf952ca9de05d82ccb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/c151a6c83a404af39ffca6a5fecd55bf, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/dc52031f68374dee973b3365276af977] to archive 2024-11-12T00:53:16,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:53:16,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/82beca1bdb3c4e28998cf9a32529f9d2 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/82beca1bdb3c4e28998cf9a32529f9d2 2024-11-12T00:53:16,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/7cf39364d2fb4299a603109a18b10ff9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/7cf39364d2fb4299a603109a18b10ff9 2024-11-12T00:53:16,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/c2ba89d85fee4d52be3497418998d020 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/c2ba89d85fee4d52be3497418998d020 2024-11-12T00:53:16,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/590740818d9f4f42a168a8e8c637117c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/590740818d9f4f42a168a8e8c637117c 2024-11-12T00:53:16,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/792347adf7ab466eadd05e62e0f03c69 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/792347adf7ab466eadd05e62e0f03c69 2024-11-12T00:53:16,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/1735cba328b3442b808e4662791d624c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/1735cba328b3442b808e4662791d624c 2024-11-12T00:53:16,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9d711b159c28423a97b4666606165356 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9d711b159c28423a97b4666606165356 2024-11-12T00:53:16,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/de8029b0b60f440da0cad50b65fd6b21 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/de8029b0b60f440da0cad50b65fd6b21 2024-11-12T00:53:16,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/764e270e3a9847b8806bffdf4bcb27de to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/764e270e3a9847b8806bffdf4bcb27de 2024-11-12T00:53:16,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/6da97efcfcbc40e7be4917cb20228b32 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/6da97efcfcbc40e7be4917cb20228b32 2024-11-12T00:53:16,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/4b9ce6f3b9a5457fa2df5a668518d29f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/4b9ce6f3b9a5457fa2df5a668518d29f 2024-11-12T00:53:16,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/11765aa085ce4491bc4aefd93f618bae to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/11765aa085ce4491bc4aefd93f618bae 2024-11-12T00:53:16,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/44192a7f65794bdb86dcb7e54fae54b2 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/44192a7f65794bdb86dcb7e54fae54b2 2024-11-12T00:53:16,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/bf21d57b9c77488e92d9d163abab33aa to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/bf21d57b9c77488e92d9d163abab33aa 2024-11-12T00:53:16,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/b3470746cae64bb6a128f2f996dc2ba6 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/b3470746cae64bb6a128f2f996dc2ba6 2024-11-12T00:53:16,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/6e9bfc64a77b43fa9fc2560ba15e0c9e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/6e9bfc64a77b43fa9fc2560ba15e0c9e 2024-11-12T00:53:16,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9ff6a8b03fd34e579887890c3eee54b9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9ff6a8b03fd34e579887890c3eee54b9 2024-11-12T00:53:16,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/46aeea6997684ce5887bbd0e5320aedc to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/46aeea6997684ce5887bbd0e5320aedc 2024-11-12T00:53:16,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/5a63fe0006034bcbbd140a14f0d01259 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/5a63fe0006034bcbbd140a14f0d01259 2024-11-12T00:53:16,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ecb3a955add048fd9c0c4dc401cd1a5f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ecb3a955add048fd9c0c4dc401cd1a5f 2024-11-12T00:53:16,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9ae217f47fe9456a8d0a10656657b822 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/9ae217f47fe9456a8d0a10656657b822 2024-11-12T00:53:16,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/f76850fbf3ce40a997ecad47b970f357 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/f76850fbf3ce40a997ecad47b970f357 2024-11-12T00:53:16,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/e526ac5850924f749031294a5fe836bc to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/e526ac5850924f749031294a5fe836bc 2024-11-12T00:53:16,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ce52b7cdd38c486e856935d41474936b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ce52b7cdd38c486e856935d41474936b 2024-11-12T00:53:16,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/32a09dd2da614faa82e45e244466c770 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/32a09dd2da614faa82e45e244466c770 2024-11-12T00:53:16,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/3214525920794454ae99ffa28dfdc5ce to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/3214525920794454ae99ffa28dfdc5ce 2024-11-12T00:53:16,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/0ea98005f161445eafe643bfdb51066f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/0ea98005f161445eafe643bfdb51066f 2024-11-12T00:53:16,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/82bc1a6387ac408b9e75b833e0e53b84 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/82bc1a6387ac408b9e75b833e0e53b84 2024-11-12T00:53:16,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/0697dde878d54eecb341a235c277ea34 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/0697dde878d54eecb341a235c277ea34 2024-11-12T00:53:16,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/bd8a6eaa594a44eba6f76ff754dc887f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/bd8a6eaa594a44eba6f76ff754dc887f 2024-11-12T00:53:16,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/4706fb1493de4b6f9209fd8cae290282 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/4706fb1493de4b6f9209fd8cae290282 2024-11-12T00:53:16,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/7e9dbfbcb78c4e8fad1aba205bdbb2d1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/7e9dbfbcb78c4e8fad1aba205bdbb2d1 2024-11-12T00:53:16,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ca3f84e21e4f4ebf952ca9de05d82ccb to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/ca3f84e21e4f4ebf952ca9de05d82ccb 2024-11-12T00:53:16,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/c151a6c83a404af39ffca6a5fecd55bf to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/c151a6c83a404af39ffca6a5fecd55bf 2024-11-12T00:53:16,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/dc52031f68374dee973b3365276af977 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/dc52031f68374dee973b3365276af977 2024-11-12T00:53:16,558 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/recovered.edits/666.seqid, newMaxSeqId=666, maxSeqId=1 2024-11-12T00:53:16,560 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. 2024-11-12T00:53:16,560 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1676): Region close journal for 8f6f5fa6d51a99bb28036e719e157e24: Waiting for close lock at 1731372795052Running coprocessor pre-close hooks at 1731372795052Disabling compacts and flushes for region at 1731372795052Disabling writes for close at 1731372795052Obtaining lock to block concurrent updates at 1731372795052Preparing flush snapshotting stores in 8f6f5fa6d51a99bb28036e719e157e24 at 1731372795052Finished memstore snapshotting TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731372795053 (+1 ms)Flushing stores of TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24. at 1731372795053Flushing 8f6f5fa6d51a99bb28036e719e157e24/A: creating writer at 1731372795053Flushing 8f6f5fa6d51a99bb28036e719e157e24/A: appending metadata at 1731372795057 (+4 ms)Flushing 8f6f5fa6d51a99bb28036e719e157e24/A: closing flushed file at 1731372795057Flushing 8f6f5fa6d51a99bb28036e719e157e24/B: creating writer at 1731372795475 (+418 ms)Flushing 8f6f5fa6d51a99bb28036e719e157e24/B: appending metadata at 1731372795478 (+3 ms)Flushing 8f6f5fa6d51a99bb28036e719e157e24/B: closing flushed file at 1731372795478Flushing 8f6f5fa6d51a99bb28036e719e157e24/C: creating writer at 1731372795896 (+418 ms)Flushing 8f6f5fa6d51a99bb28036e719e157e24/C: appending metadata at 1731372795899 (+3 ms)Flushing 8f6f5fa6d51a99bb28036e719e157e24/C: closing flushed file at 1731372795900 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f990432: reopening flushed file at 1731372796310 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d46c263: reopening flushed file at 1731372796317 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49d380e9: reopening flushed file at 1731372796325 (+8 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8f6f5fa6d51a99bb28036e719e157e24 in 1283ms, sequenceid=663, compaction requested=true at 1731372796335 (+10 ms)Writing region close event to WAL at 1731372796554 (+219 ms)Running coprocessor post-close hooks at 1731372796559 (+5 ms)Closed at 1731372796559 2024-11-12T00:53:16,564 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=8f6f5fa6d51a99bb28036e719e157e24, regionState=CLOSED 2024-11-12T00:53:16,565 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(157): Closed 8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:16,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:53:16,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-12T00:53:16,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseRegionProcedure 8f6f5fa6d51a99bb28036e719e157e24, server=6928b69dca1b,33953,1731372765559 in 1.6750 sec 2024-11-12T00:53:16,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-12T00:53:16,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f6f5fa6d51a99bb28036e719e157e24, UNASSIGN in 1.6870 sec 2024-11-12T00:53:16,578 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-12T00:53:16,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6970 sec 2024-11-12T00:53:16,581 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372796581"}]},"ts":"1731372796581"} 2024-11-12T00:53:16,583 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T00:53:16,583 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T00:53:16,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7180 sec 2024-11-12T00:53:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T00:53:17,036 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:17,038 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T00:53:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:17,043 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T00:53:17,045 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=23, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:17,050 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:17,055 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/recovered.edits] 2024-11-12T00:53:17,058 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/08b8dda2f5ae41299c0dbb1871f3ccb0 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/08b8dda2f5ae41299c0dbb1871f3ccb0 2024-11-12T00:53:17,060 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/e971eb1732224a4c898fc92a1b1a0ba7 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/e971eb1732224a4c898fc92a1b1a0ba7 2024-11-12T00:53:17,061 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/fe6746cd1396441baef926e7dd448a73 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/A/fe6746cd1396441baef926e7dd448a73 2024-11-12T00:53:17,063 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/281a6195c7234c4da15579bb2e6275c4 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/281a6195c7234c4da15579bb2e6275c4 2024-11-12T00:53:17,064 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/74d4d704fb594a31b0acc00d05e0e48e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/74d4d704fb594a31b0acc00d05e0e48e 2024-11-12T00:53:17,066 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/80c2316dfd7a490c9d5e1c362ae4eb8e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/B/80c2316dfd7a490c9d5e1c362ae4eb8e 2024-11-12T00:53:17,068 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/1fc7afedfb334288beb59b0712f47bf2 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/1fc7afedfb334288beb59b0712f47bf2 2024-11-12T00:53:17,069 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/2dbe29f4c114465f9984ca6d75305883 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/2dbe29f4c114465f9984ca6d75305883 2024-11-12T00:53:17,070 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/decd936d75f2430aacb1bee8af3d0921 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/C/decd936d75f2430aacb1bee8af3d0921 2024-11-12T00:53:17,073 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/recovered.edits/666.seqid to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24/recovered.edits/666.seqid 2024-11-12T00:53:17,073 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/8f6f5fa6d51a99bb28036e719e157e24 2024-11-12T00:53:17,073 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T00:53:17,079 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=23, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:17,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-12T00:53:17,090 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T00:53:17,126 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T00:53:17,127 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=23, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:17,128 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T00:53:17,128 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731372797128"}]},"ts":"9223372036854775807"} 2024-11-12T00:53:17,132 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T00:53:17,132 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8f6f5fa6d51a99bb28036e719e157e24, NAME => 'TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T00:53:17,133 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T00:53:17,133 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731372797133"}]},"ts":"9223372036854775807"} 2024-11-12T00:53:17,136 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T00:53:17,138 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=23, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:17,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 99 msec 2024-11-12T00:53:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T00:53:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T00:53:17,157 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:17,170 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1762401511_22 at /127.0.0.1:33808 [Waiting for operation #315] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1380178899_22 at /127.0.0.1:41330 [Waiting for operation #329] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1380178899_22 at /127.0.0.1:48426 [Waiting for operation #67] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;6928b69dca1b:33953-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=532 (was 479) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5669 (was 6292) 2024-11-12T00:53:17,181 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=532, ProcessCount=11, AvailableMemoryMB=5668 2024-11-12T00:53:17,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T00:53:17,184 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T00:53:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:17,187 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T00:53:17,187 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:17,187 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 24 2024-11-12T00:53:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-12T00:53:17,189 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T00:53:17,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741951_1127 (size=1061) 2024-11-12T00:53:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-12T00:53:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-12T00:53:17,599 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6 2024-11-12T00:53:17,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741952_1128 (size=53) 2024-11-12T00:53:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-12T00:53:18,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:53:18,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 335e77c42adcd1ca9b481da0f68cba79, disabling compactions & flushes 2024-11-12T00:53:18,013 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:18,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:18,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. after waiting 0 ms 2024-11-12T00:53:18,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:18,013 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:18,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 335e77c42adcd1ca9b481da0f68cba79: Waiting for close lock at 1731372798013Disabling compacts and flushes for region at 1731372798013Disabling writes for close at 1731372798013Writing region close event to WAL at 1731372798013Closed at 1731372798013 2024-11-12T00:53:18,015 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T00:53:18,015 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731372798015"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731372798015"}]},"ts":"1731372798015"} 2024-11-12T00:53:18,018 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T00:53:18,019 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T00:53:18,020 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372798019"}]},"ts":"1731372798019"} 2024-11-12T00:53:18,022 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T00:53:18,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=335e77c42adcd1ca9b481da0f68cba79, ASSIGN}] 2024-11-12T00:53:18,024 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=335e77c42adcd1ca9b481da0f68cba79, ASSIGN 2024-11-12T00:53:18,025 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=335e77c42adcd1ca9b481da0f68cba79, ASSIGN; state=OFFLINE, location=6928b69dca1b,33953,1731372765559; forceNewPlan=false, retain=false 2024-11-12T00:53:18,176 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=335e77c42adcd1ca9b481da0f68cba79, regionState=OPENING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:18,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=335e77c42adcd1ca9b481da0f68cba79, ASSIGN because future has completed 2024-11-12T00:53:18,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:53:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-12T00:53:18,334 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:18,334 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7752): Opening region: {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} 2024-11-12T00:53:18,335 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:18,335 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:53:18,335 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7794): checking encryption for 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:18,335 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7797): checking classloading for 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:18,336 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:18,337 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:53:18,338 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 335e77c42adcd1ca9b481da0f68cba79 columnFamilyName A 2024-11-12T00:53:18,338 DEBUG [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:18,339 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.HStore(327): Store=335e77c42adcd1ca9b481da0f68cba79/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:53:18,339 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:18,340 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:53:18,340 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 335e77c42adcd1ca9b481da0f68cba79 columnFamilyName B 2024-11-12T00:53:18,340 DEBUG [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:18,341 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.HStore(327): Store=335e77c42adcd1ca9b481da0f68cba79/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:53:18,341 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:18,342 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:53:18,342 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 335e77c42adcd1ca9b481da0f68cba79 columnFamilyName C 2024-11-12T00:53:18,342 DEBUG [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:18,343 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.HStore(327): Store=335e77c42adcd1ca9b481da0f68cba79/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:53:18,343 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:18,343 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1038): replaying wal for 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:18,343 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:18,344 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:18,345 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1048): stopping wal replay for 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:18,345 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1060): Cleaning up temporary data for 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:18,345 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T00:53:18,346 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): writing seq id for 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:18,349 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T00:53:18,349 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1114): Opened 335e77c42adcd1ca9b481da0f68cba79; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73754325, jitterRate=0.09902508556842804}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T00:53:18,349 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:18,350 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1006): Region open journal for 335e77c42adcd1ca9b481da0f68cba79: Running coprocessor pre-open hook at 1731372798335Writing region info on filesystem at 1731372798335Initializing all the Stores at 1731372798336 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372798336Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372798336Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372798336Cleaning up temporary data from old regions at 1731372798345 (+9 ms)Running coprocessor post-open hooks at 1731372798349 (+4 ms)Region opened successfully at 1731372798350 (+1 ms) 2024-11-12T00:53:18,351 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., pid=26, masterSystemTime=1731372798331 2024-11-12T00:53:18,353 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:18,353 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:18,353 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=335e77c42adcd1ca9b481da0f68cba79, regionState=OPEN, openSeqNum=2, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:18,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:53:18,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-12T00:53:18,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 in 178 msec 2024-11-12T00:53:18,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-12T00:53:18,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=335e77c42adcd1ca9b481da0f68cba79, ASSIGN in 336 msec 2024-11-12T00:53:18,361 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T00:53:18,361 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372798361"}]},"ts":"1731372798361"} 2024-11-12T00:53:18,363 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T00:53:18,363 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T00:53:18,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-12T00:53:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-12T00:53:19,336 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:19,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ff3539, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:19,337 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:19,337 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:19,339 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c6a2ed4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:19,341 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:19,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aafdf29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:19,342 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:19,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:19,346 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T00:53:19,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 6928b69dca1b,45925,1731372764943 2024-11-12T00:53:19,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c1b1db5 2024-11-12T00:53:19,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T00:53:19,348 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T00:53:19,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T00:53:19,354 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T00:53:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:19,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741953_1129 (size=1097) 2024-11-12T00:53:19,778 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T00:53:19,779 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T00:53:19,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T00:53:19,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=335e77c42adcd1ca9b481da0f68cba79, REOPEN/MOVE}] 2024-11-12T00:53:19,793 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=335e77c42adcd1ca9b481da0f68cba79, REOPEN/MOVE 2024-11-12T00:53:19,793 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=335e77c42adcd1ca9b481da0f68cba79, regionState=CLOSING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:19,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=335e77c42adcd1ca9b481da0f68cba79, REOPEN/MOVE because future has completed 2024-11-12T00:53:19,796 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T00:53:19,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:53:19,949 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:19,949 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T00:53:19,949 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 335e77c42adcd1ca9b481da0f68cba79, disabling compactions & flushes 2024-11-12T00:53:19,949 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:19,949 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:19,949 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. after waiting 0 ms 2024-11-12T00:53:19,950 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:19,954 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T00:53:19,955 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:19,955 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 335e77c42adcd1ca9b481da0f68cba79: Waiting for close lock at 1731372799949Running coprocessor pre-close hooks at 1731372799949Disabling compacts and flushes for region at 1731372799949Disabling writes for close at 1731372799949Writing region close event to WAL at 1731372799950 (+1 ms)Running coprocessor post-close hooks at 1731372799955 (+5 ms)Closed at 1731372799955 2024-11-12T00:53:19,955 WARN [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionServer(3299): Not adding moved region record: 335e77c42adcd1ca9b481da0f68cba79 to self. 2024-11-12T00:53:19,957 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:19,957 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=335e77c42adcd1ca9b481da0f68cba79, regionState=CLOSED 2024-11-12T00:53:19,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:53:19,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-12T00:53:19,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 in 163 msec 2024-11-12T00:53:19,962 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=335e77c42adcd1ca9b481da0f68cba79, REOPEN/MOVE; state=CLOSED, location=6928b69dca1b,33953,1731372765559; forceNewPlan=false, retain=true 2024-11-12T00:53:20,113 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=335e77c42adcd1ca9b481da0f68cba79, regionState=OPENING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=335e77c42adcd1ca9b481da0f68cba79, REOPEN/MOVE because future has completed 2024-11-12T00:53:20,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:53:20,282 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:20,283 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} 2024-11-12T00:53:20,284 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,284 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:53:20,284 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,284 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,288 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,289 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:53:20,295 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 335e77c42adcd1ca9b481da0f68cba79 columnFamilyName A 2024-11-12T00:53:20,297 DEBUG [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:20,298 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.HStore(327): Store=335e77c42adcd1ca9b481da0f68cba79/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:53:20,298 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,299 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:53:20,299 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 335e77c42adcd1ca9b481da0f68cba79 columnFamilyName B 2024-11-12T00:53:20,299 DEBUG [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:20,299 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.HStore(327): Store=335e77c42adcd1ca9b481da0f68cba79/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:53:20,300 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,300 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:53:20,300 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 335e77c42adcd1ca9b481da0f68cba79 columnFamilyName C 2024-11-12T00:53:20,300 DEBUG [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:20,301 INFO [StoreOpener-335e77c42adcd1ca9b481da0f68cba79-1 {}] regionserver.HStore(327): Store=335e77c42adcd1ca9b481da0f68cba79/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:53:20,301 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:20,301 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,301 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,303 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,304 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,304 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,304 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T00:53:20,306 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,307 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 335e77c42adcd1ca9b481da0f68cba79; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64317699, jitterRate=-0.04159159958362579}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T00:53:20,307 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,307 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 335e77c42adcd1ca9b481da0f68cba79: Running coprocessor pre-open hook at 1731372800285Writing region info on filesystem at 1731372800285Initializing all the Stores at 1731372800286 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372800286Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372800287 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372800287Cleaning up temporary data from old regions at 1731372800304 (+17 ms)Running coprocessor post-open hooks at 1731372800307 (+3 ms)Region opened successfully at 1731372800307 2024-11-12T00:53:20,308 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., pid=31, masterSystemTime=1731372800273 2024-11-12T00:53:20,310 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:20,310 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:20,311 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=335e77c42adcd1ca9b481da0f68cba79, regionState=OPEN, openSeqNum=5, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:53:20,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-12T00:53:20,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 in 196 msec 2024-11-12T00:53:20,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-12T00:53:20,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=335e77c42adcd1ca9b481da0f68cba79, REOPEN/MOVE in 524 msec 2024-11-12T00:53:20,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-12T00:53:20,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 534 msec 2024-11-12T00:53:20,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 967 msec 2024-11-12T00:53:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T00:53:20,327 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:20,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323fc298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,328 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:20,328 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:20,329 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:20,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6863a4d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:20,331 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b0ecd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb0fd71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:20,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:20,334 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@200a7802, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:20,336 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79cc0d53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,338 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:20,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:20,339 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e408df5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:20,341 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b6bb510, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:20,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:20,344 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:20,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e87ff64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:20,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:20,347 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@92784e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,348 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:20,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:20,350 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1c830d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:20,352 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea7deff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,354 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:20,354 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:20,356 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b59c42b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:20,358 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7194e492, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677aa7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,360 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:20,360 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:20,361 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd611b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:20,364 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2441aae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,365 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:20,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:20,366 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62a9f068, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:20,369 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b1266e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,370 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:20,371 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:20,372 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f3f5851, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:20,374 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1389b68a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:20,375 DEBUG [Thread-653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:20,378 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:20,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:20,381 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:53:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:53:20,394 DEBUG [Thread-655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:20,394 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,396 DEBUG [Thread-655 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:20,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T00:53:20,397 DEBUG [Thread-655 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:20,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:20,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:20,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:20,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:20,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:20,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:20,398 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:20,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:20,400 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:20,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:20,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:20,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:20,402 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:53:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:53:20,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:20,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:53:20,407 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:53:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:20,409 DEBUG [Thread-664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:20,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:20,410 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:20,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:20,413 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:20,414 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:20,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:53:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees 2024-11-12T00:53:20,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:20,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T00:53:20,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:20,418 DEBUG [Thread-647 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:53:20,418 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:53:20,421 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:20,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:20,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111293c8b6d08b40434ca2d98aca284d4b5a_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372800393/Put/seqid=0 2024-11-12T00:53:20,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372860456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731372800423, max timestamp=1731372800459Num uniques 859; 2024-11-12T00:53:20,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T00:53:20,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731372800423, max timestamp=1731372800459Num uniques 859; 2024-11-12T00:53:20,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T00:53:20,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372800452, max timestamp=1731372800459Num uniques 859; 2024-11-12T00:53:20,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:53:20,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741954_1130 (size=12154) 2024-11-12T00:53:20,479 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:20,485 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111293c8b6d08b40434ca2d98aca284d4b5a_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111293c8b6d08b40434ca2d98aca284d4b5a_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,487 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/89e747e5f8bf4e9b8dd91aac552b57d8, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:20,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/89e747e5f8bf4e9b8dd91aac552b57d8 is 175, key is test_row_0/A:col10/1731372800393/Put/seqid=0 2024-11-12T00:53:20,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741955_1131 (size=30955) 2024-11-12T00:53:20,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372860497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372860500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52768 deadline: 1731372860511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372860514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,519 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/89e747e5f8bf4e9b8dd91aac552b57d8 2024-11-12T00:53:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T00:53:20,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/05add621e4444b5e8772717538ce35f9 is 50, key is test_row_0/B:col10/1731372800393/Put/seqid=0 2024-11-12T00:53:20,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741956_1132 (size=12001) 2024-11-12T00:53:20,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/05add621e4444b5e8772717538ce35f9 2024-11-12T00:53:20,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-12T00:53:20,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:20,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:20,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:20,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:20,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:20,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372860566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/0c5fdc1ff1ee4021bc2d71a4178170de is 50, key is test_row_0/C:col10/1731372800393/Put/seqid=0 2024-11-12T00:53:20,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741957_1133 (size=12001) 2024-11-12T00:53:20,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/0c5fdc1ff1ee4021bc2d71a4178170de 2024-11-12T00:53:20,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/89e747e5f8bf4e9b8dd91aac552b57d8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/89e747e5f8bf4e9b8dd91aac552b57d8 2024-11-12T00:53:20,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/89e747e5f8bf4e9b8dd91aac552b57d8, entries=150, sequenceid=16, filesize=30.2 K 2024-11-12T00:53:20,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372860625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372860626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52768 deadline: 1731372860626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372860626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/05add621e4444b5e8772717538ce35f9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/05add621e4444b5e8772717538ce35f9 2024-11-12T00:53:20,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/05add621e4444b5e8772717538ce35f9, entries=150, sequenceid=16, filesize=11.7 K 2024-11-12T00:53:20,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/0c5fdc1ff1ee4021bc2d71a4178170de as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/0c5fdc1ff1ee4021bc2d71a4178170de 2024-11-12T00:53:20,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/0c5fdc1ff1ee4021bc2d71a4178170de, entries=150, sequenceid=16, filesize=11.7 K 2024-11-12T00:53:20,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 335e77c42adcd1ca9b481da0f68cba79 in 263ms, sequenceid=16, compaction requested=false 2024-11-12T00:53:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:20,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-12T00:53:20,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:20,732 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=120.76 KB heapSize=335.77 KB 2024-11-12T00:53:20,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:20,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:20,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:20,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:20,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:20,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T00:53:20,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112123867a816ee433183a87c1f199ed606_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372800508/Put/seqid=0 2024-11-12T00:53:20,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741958_1134 (size=12154) 2024-11-12T00:53:20,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:20,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52768 deadline: 1731372860846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372860846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372860847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372860847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372860858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372860965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:20,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372860966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:20,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:20,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T00:53:21,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:21,166 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112123867a816ee433183a87c1f199ed606_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112123867a816ee433183a87c1f199ed606_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:21,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52768 deadline: 1731372861165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:21,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372861166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:21,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/92a9eeda02a7428ca9268f7b388c7aa9, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:21,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:21,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372861167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:21,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/92a9eeda02a7428ca9268f7b388c7aa9 is 175, key is test_row_0/A:col10/1731372800508/Put/seqid=0 2024-11-12T00:53:21,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:21,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372861185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:21,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372861185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:21,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741959_1135 (size=30955) 2024-11-12T00:53:21,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372861496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:21,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:21,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372861496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T00:53:21,604 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/92a9eeda02a7428ca9268f7b388c7aa9 2024-11-12T00:53:21,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/451bb14153484514bc2fc6ca3b33b23b is 50, key is test_row_0/B:col10/1731372800508/Put/seqid=0 2024-11-12T00:53:21,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741960_1136 (size=12001) 2024-11-12T00:53:21,633 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/451bb14153484514bc2fc6ca3b33b23b 2024-11-12T00:53:21,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/22f361dc6c484a29affe5b477ee90ebe is 50, key is test_row_0/C:col10/1731372800508/Put/seqid=0 2024-11-12T00:53:21,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741961_1137 (size=12001) 2024-11-12T00:53:21,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372861676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:21,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372861676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:21,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52768 deadline: 1731372861676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:22,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372862006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:22,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:22,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:22,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:22,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372862006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:22,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:22,071 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/22f361dc6c484a29affe5b477ee90ebe 2024-11-12T00:53:22,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/92a9eeda02a7428ca9268f7b388c7aa9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/92a9eeda02a7428ca9268f7b388c7aa9 2024-11-12T00:53:22,083 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/92a9eeda02a7428ca9268f7b388c7aa9, entries=150, sequenceid=49, filesize=30.2 K 2024-11-12T00:53:22,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/451bb14153484514bc2fc6ca3b33b23b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/451bb14153484514bc2fc6ca3b33b23b 2024-11-12T00:53:22,093 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/451bb14153484514bc2fc6ca3b33b23b, entries=150, sequenceid=49, filesize=11.7 K 2024-11-12T00:53:22,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/22f361dc6c484a29affe5b477ee90ebe as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/22f361dc6c484a29affe5b477ee90ebe 2024-11-12T00:53:22,102 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/22f361dc6c484a29affe5b477ee90ebe, entries=150, sequenceid=49, filesize=11.7 K 2024-11-12T00:53:22,103 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~335.72 KB/343776, currentSize=60.38 KB/61830 for 335e77c42adcd1ca9b481da0f68cba79 in 1371ms, sequenceid=49, compaction requested=false 2024-11-12T00:53:22,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:22,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:22,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-12T00:53:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-12T00:53:22,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-12T00:53:22,109 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T00:53:22,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6850 sec 2024-11-12T00:53:22,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees in 1.6960 sec 2024-11-12T00:53:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T00:53:22,584 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:22,585 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-12T00:53:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T00:53:22,592 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:22,593 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:22,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:22,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T00:53:22,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:22,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:22,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:22,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:22,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:22,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T00:53:22,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fdf9560943e941258331a66f203d09dc_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372800840/Put/seqid=0 2024-11-12T00:53:22,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741962_1138 (size=14594) 2024-11-12T00:53:22,723 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:22,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372862730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:22,732 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fdf9560943e941258331a66f203d09dc_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112fdf9560943e941258331a66f203d09dc_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:22,733 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/43ddd530356945ba8b6d2a90b4a696f5, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:22,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/43ddd530356945ba8b6d2a90b4a696f5 is 175, key is test_row_0/A:col10/1731372800840/Put/seqid=0 2024-11-12T00:53:22,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52768 deadline: 1731372862732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:22,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372862732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:22,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741963_1139 (size=39549) 2024-11-12T00:53:22,744 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/43ddd530356945ba8b6d2a90b4a696f5 2024-11-12T00:53:22,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T00:53:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:22,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:22,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:22,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:22,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/7b15e6cc56014882936ae7563f4ece35 is 50, key is test_row_0/B:col10/1731372800840/Put/seqid=0 2024-11-12T00:53:22,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741964_1140 (size=12001) 2024-11-12T00:53:22,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/7b15e6cc56014882936ae7563f4ece35 2024-11-12T00:53:22,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/b02914e06e354374b37f2b080b28e63e is 50, key is test_row_0/C:col10/1731372800840/Put/seqid=0 2024-11-12T00:53:22,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741965_1141 (size=12001) 2024-11-12T00:53:22,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/b02914e06e354374b37f2b080b28e63e 2024-11-12T00:53:22,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/43ddd530356945ba8b6d2a90b4a696f5 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/43ddd530356945ba8b6d2a90b4a696f5 2024-11-12T00:53:22,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/43ddd530356945ba8b6d2a90b4a696f5, entries=200, sequenceid=62, filesize=38.6 K 2024-11-12T00:53:22,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/7b15e6cc56014882936ae7563f4ece35 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/7b15e6cc56014882936ae7563f4ece35 2024-11-12T00:53:22,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/7b15e6cc56014882936ae7563f4ece35, entries=150, sequenceid=62, filesize=11.7 K 2024-11-12T00:53:22,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/b02914e06e354374b37f2b080b28e63e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/b02914e06e354374b37f2b080b28e63e 2024-11-12T00:53:22,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372862836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:22,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/b02914e06e354374b37f2b080b28e63e, entries=150, sequenceid=62, filesize=11.7 K 2024-11-12T00:53:22,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 335e77c42adcd1ca9b481da0f68cba79 in 162ms, sequenceid=62, compaction requested=true 2024-11-12T00:53:22,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:22,853 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:22,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:53:22,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:22,854 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:22,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:53:22,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:22,855 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:22,855 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/A is initiating minor compaction (all files) 2024-11-12T00:53:22,855 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/A in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:22,855 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/89e747e5f8bf4e9b8dd91aac552b57d8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/92a9eeda02a7428ca9268f7b388c7aa9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/43ddd530356945ba8b6d2a90b4a696f5] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=99.1 K 2024-11-12T00:53:22,855 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:22,856 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/89e747e5f8bf4e9b8dd91aac552b57d8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/92a9eeda02a7428ca9268f7b388c7aa9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/43ddd530356945ba8b6d2a90b4a696f5] 2024-11-12T00:53:22,856 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:22,856 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/B is initiating minor compaction (all files) 2024-11-12T00:53:22,856 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/B in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:22,856 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/05add621e4444b5e8772717538ce35f9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/451bb14153484514bc2fc6ca3b33b23b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/7b15e6cc56014882936ae7563f4ece35] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=35.2 K 2024-11-12T00:53:22,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731372802726, max timestamp=1731372802853Num uniques 859; 2024-11-12T00:53:22,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:53:22,858 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89e747e5f8bf4e9b8dd91aac552b57d8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731372800387 2024-11-12T00:53:22,858 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 05add621e4444b5e8772717538ce35f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731372800387 2024-11-12T00:53:22,858 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92a9eeda02a7428ca9268f7b388c7aa9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731372800494 2024-11-12T00:53:22,858 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 451bb14153484514bc2fc6ca3b33b23b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731372800494 2024-11-12T00:53:22,859 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b15e6cc56014882936ae7563f4ece35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731372800797 2024-11-12T00:53:22,859 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43ddd530356945ba8b6d2a90b4a696f5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731372800797 2024-11-12T00:53:22,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372802726, max timestamp=1731372802853Num uniques 859; 2024-11-12T00:53:22,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:53:22,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372802726, max timestamp=1731372802853Num uniques 859; 2024-11-12T00:53:22,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:53:22,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:53:22,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:22,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=35.78 KB heapSize=91.23 KB 2024-11-12T00:53:22,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:22,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:22,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:22,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:22,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:22,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:22,868 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:22,881 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:22,881 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/4789914765cf46daaa5c6ba84e0ecfa0 is 50, key is test_row_0/B:col10/1731372800840/Put/seqid=0 2024-11-12T00:53:22,897 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411120d1a1126a3c947b7b3c85033ed02251d_335e77c42adcd1ca9b481da0f68cba79 store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:22,902 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411120d1a1126a3c947b7b3c85033ed02251d_335e77c42adcd1ca9b481da0f68cba79, store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:22,902 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120d1a1126a3c947b7b3c85033ed02251d_335e77c42adcd1ca9b481da0f68cba79 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:22,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T00:53:22,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:22,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:22,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:22,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:22,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T00:53:22,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b281caff91684412b5c0443637809861_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372802853/Put/seqid=0 2024-11-12T00:53:22,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741966_1142 (size=12104) 2024-11-12T00:53:22,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741968_1144 (size=17034) 2024-11-12T00:53:22,961 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/4789914765cf46daaa5c6ba84e0ecfa0 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/4789914765cf46daaa5c6ba84e0ecfa0 2024-11-12T00:53:22,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741967_1143 (size=4469) 2024-11-12T00:53:22,974 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#A#compaction#123 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:22,974 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/0eca9cc4c18d41538456eb604ca260c4 is 175, key is test_row_0/A:col10/1731372800840/Put/seqid=0 2024-11-12T00:53:22,978 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/B of 335e77c42adcd1ca9b481da0f68cba79 into 4789914765cf46daaa5c6ba84e0ecfa0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:22,978 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:22,978 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/B, priority=13, startTime=1731372802853; duration=0sec 2024-11-12T00:53:22,979 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:22,979 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:B 2024-11-12T00:53:22,979 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:22,988 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:22,988 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/C is initiating minor compaction (all files) 2024-11-12T00:53:22,989 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/C in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:22,989 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/0c5fdc1ff1ee4021bc2d71a4178170de, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/22f361dc6c484a29affe5b477ee90ebe, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/b02914e06e354374b37f2b080b28e63e] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=35.2 K 2024-11-12T00:53:22,989 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c5fdc1ff1ee4021bc2d71a4178170de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731372800387 2024-11-12T00:53:22,990 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 22f361dc6c484a29affe5b477ee90ebe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731372800494 2024-11-12T00:53:22,990 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting b02914e06e354374b37f2b080b28e63e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731372800797 2024-11-12T00:53:23,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731372802926, max timestamp=1731372802997Num uniques 859; 2024-11-12T00:53:23,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T00:53:23,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731372802926, max timestamp=1731372802997Num uniques 859; 2024-11-12T00:53:23,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T00:53:23,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731372802926, max timestamp=1731372802997Num uniques 859; 2024-11-12T00:53:23,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T00:53:23,009 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:23,010 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/3312e7eb3c9041669e15bc8b2a60b6dc is 50, key is test_row_0/C:col10/1731372800840/Put/seqid=0 2024-11-12T00:53:23,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741969_1145 (size=31058) 2024-11-12T00:53:23,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741970_1146 (size=12104) 2024-11-12T00:53:23,051 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/3312e7eb3c9041669e15bc8b2a60b6dc as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/3312e7eb3c9041669e15bc8b2a60b6dc 2024-11-12T00:53:23,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T00:53:23,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:23,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:23,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:23,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52768 deadline: 1731372863050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372863052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372863054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731372803038, max timestamp=1731372803059Num uniques 1118; 2024-11-12T00:53:23,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=16848, suffix heap size=144240,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=368 2024-11-12T00:53:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731372803038, max timestamp=1731372803059Num uniques 1118; 2024-11-12T00:53:23,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731372803038, max timestamp=1731372803059Num uniques 1118; 2024-11-12T00:53:23,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=16848, suffix heap size=144240,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=368 2024-11-12T00:53:23,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=16848, suffix heap size=144240,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=368 2024-11-12T00:53:23,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,067 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/C of 335e77c42adcd1ca9b481da0f68cba79 into 3312e7eb3c9041669e15bc8b2a60b6dc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:23,068 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:23,069 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/C, priority=13, startTime=1731372802864; duration=0sec 2024-11-12T00:53:23,069 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:23,069 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:C 2024-11-12T00:53:23,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372863122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372863124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372863166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52768 deadline: 1731372863178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372863178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T00:53:23,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T00:53:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:23,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372863236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372863238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,362 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:23,368 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b281caff91684412b5c0443637809861_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b281caff91684412b5c0443637809861_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:23,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T00:53:23,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:23,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:23,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:23,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,369 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/f6b0b36ba70a4270905bcec7602132d8, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:23,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/f6b0b36ba70a4270905bcec7602132d8 is 175, key is test_row_0/A:col10/1731372802853/Put/seqid=0 2024-11-12T00:53:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741971_1147 (size=48139) 2024-11-12T00:53:23,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372863375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52768 deadline: 1731372863386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372863386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,431 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/0eca9cc4c18d41538456eb604ca260c4 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/0eca9cc4c18d41538456eb604ca260c4 2024-11-12T00:53:23,444 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/A of 335e77c42adcd1ca9b481da0f68cba79 into 0eca9cc4c18d41538456eb604ca260c4(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:23,444 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:23,444 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/A, priority=13, startTime=1731372802853; duration=0sec 2024-11-12T00:53:23,445 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:23,446 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:A 2024-11-12T00:53:23,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372863445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372863446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T00:53:23,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:23,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:23,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:23,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T00:53:23,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:23,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:23,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:23,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372863685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52768 deadline: 1731372863706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372863706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T00:53:23,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372863756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372863765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:23,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:23,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:23,780 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/f6b0b36ba70a4270905bcec7602132d8 2024-11-12T00:53:23,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/42c0b8c9fae64e728acefeb76fb3a3fb is 50, key is test_row_0/B:col10/1731372802853/Put/seqid=0 2024-11-12T00:53:23,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T00:53:23,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:23,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:23,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741972_1148 (size=12001) 2024-11-12T00:53:23,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/42c0b8c9fae64e728acefeb76fb3a3fb 2024-11-12T00:53:23,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/8388ca7b23e14c909c4ccfaa7631bc52 is 50, key is test_row_0/C:col10/1731372802853/Put/seqid=0 2024-11-12T00:53:23,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741973_1149 (size=12001) 2024-11-12T00:53:23,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/8388ca7b23e14c909c4ccfaa7631bc52 2024-11-12T00:53:23,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/f6b0b36ba70a4270905bcec7602132d8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/f6b0b36ba70a4270905bcec7602132d8 2024-11-12T00:53:23,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/f6b0b36ba70a4270905bcec7602132d8, entries=250, sequenceid=84, filesize=47.0 K 2024-11-12T00:53:23,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/42c0b8c9fae64e728acefeb76fb3a3fb as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/42c0b8c9fae64e728acefeb76fb3a3fb 2024-11-12T00:53:23,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/42c0b8c9fae64e728acefeb76fb3a3fb, entries=150, sequenceid=84, filesize=11.7 K 2024-11-12T00:53:23,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/8388ca7b23e14c909c4ccfaa7631bc52 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/8388ca7b23e14c909c4ccfaa7631bc52 2024-11-12T00:53:23,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/8388ca7b23e14c909c4ccfaa7631bc52, entries=150, sequenceid=84, filesize=11.7 K 2024-11-12T00:53:23,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~35.78 KB/36640, heapSize ~91.19 KB/93376, currentSize=154.31 KB/158010 for 335e77c42adcd1ca9b481da0f68cba79 in 1084ms, sequenceid=84, compaction requested=false 2024-11-12T00:53:23,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:23,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T00:53:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:23,989 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=154.31 KB heapSize=425.16 KB 2024-11-12T00:53:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:24,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c91d6ab3e6174f53833bbe43f140c91f_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372803122/Put/seqid=0 2024-11-12T00:53:24,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741974_1150 (size=12304) 2024-11-12T00:53:24,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,039 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c91d6ab3e6174f53833bbe43f140c91f_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c91d6ab3e6174f53833bbe43f140c91f_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:24,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/6e5be26bb62149ec8bb2c95b94d7996e, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:24,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/6e5be26bb62149ec8bb2c95b94d7996e is 175, key is test_row_0/A:col10/1731372803122/Put/seqid=0 2024-11-12T00:53:24,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741975_1151 (size=31105) 2024-11-12T00:53:24,054 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/6e5be26bb62149ec8bb2c95b94d7996e 2024-11-12T00:53:24,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/c311faf738e742c9a297534514e09c26 is 50, key is test_row_0/B:col10/1731372803122/Put/seqid=0 2024-11-12T00:53:24,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741976_1152 (size=12151) 2024-11-12T00:53:24,083 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/c311faf738e742c9a297534514e09c26 2024-11-12T00:53:24,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/355be2668fcc4aaea7b57aa2d591cca3 is 50, key is test_row_0/C:col10/1731372803122/Put/seqid=0 2024-11-12T00:53:24,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741977_1153 (size=12151) 2024-11-12T00:53:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:24,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:24,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:24,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372864214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:24,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52768 deadline: 1731372864215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:24,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:24,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372864226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:24,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:24,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:24,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:24,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372864275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:24,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372864275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:24,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372864326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:24,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:24,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:24,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:24,504 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/355be2668fcc4aaea7b57aa2d591cca3 2024-11-12T00:53:24,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/6e5be26bb62149ec8bb2c95b94d7996e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/6e5be26bb62149ec8bb2c95b94d7996e 2024-11-12T00:53:24,515 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/6e5be26bb62149ec8bb2c95b94d7996e, entries=150, sequenceid=140, filesize=30.4 K 2024-11-12T00:53:24,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/c311faf738e742c9a297534514e09c26 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/c311faf738e742c9a297534514e09c26 2024-11-12T00:53:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,523 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/c311faf738e742c9a297534514e09c26, entries=150, sequenceid=140, filesize=11.9 K 2024-11-12T00:53:24,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/355be2668fcc4aaea7b57aa2d591cca3 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/355be2668fcc4aaea7b57aa2d591cca3 2024-11-12T00:53:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,554 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/355be2668fcc4aaea7b57aa2d591cca3, entries=150, sequenceid=140, filesize=11.9 K 2024-11-12T00:53:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,555 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.11 KB/435312, currentSize=33.54 KB/34350 for 335e77c42adcd1ca9b481da0f68cba79 in 566ms, sequenceid=140, compaction requested=true 2024-11-12T00:53:24,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:24,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:24,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-12T00:53:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-12T00:53:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-12T00:53:24,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9640 sec 2024-11-12T00:53:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 1.9770 sec 2024-11-12T00:53:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:24,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T00:53:24,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:24,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:24,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:24,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:24,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:24,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112562c1a091e4f45cf87f3ebd7124ae481_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_1/A:col10/1731372804212/Put/seqid=0 2024-11-12T00:53:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741978_1154 (size=14794) 2024-11-12T00:53:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,651 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,657 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112562c1a091e4f45cf87f3ebd7124ae481_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112562c1a091e4f45cf87f3ebd7124ae481_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,660 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/9d10d39fb7fb45609e231947ecafc1f2, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/9d10d39fb7fb45609e231947ecafc1f2 is 175, key is test_row_1/A:col10/1731372804212/Put/seqid=0 2024-11-12T00:53:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741979_1155 (size=39745) 2024-11-12T00:53:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,725 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=150, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/9d10d39fb7fb45609e231947ecafc1f2 2024-11-12T00:53:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T00:53:24,746 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/18d9f6d7e41f4ad6930c56da166b19d4 is 50, key is test_row_1/B:col10/1731372804212/Put/seqid=0 2024-11-12T00:53:24,748 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-12T00:53:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,750 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T00:53:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,753 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:24,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:24,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741980_1156 (size=9757) 2024-11-12T00:53:24,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/18d9f6d7e41f4ad6930c56da166b19d4 2024-11-12T00:53:24,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/0a103485fd5e40c3a014c9963c4840c7 is 50, key is test_row_1/C:col10/1731372804212/Put/seqid=0 2024-11-12T00:53:24,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741981_1157 (size=9757) 2024-11-12T00:53:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T00:53:24,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372804838, max timestamp=1731372804890Num uniques 859; 2024-11-12T00:53:24,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:53:24,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372804838, max timestamp=1731372804890Num uniques 859; 2024-11-12T00:53:24,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372804838, max timestamp=1731372804890Num uniques 859; 2024-11-12T00:53:24,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:53:24,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:53:24,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T00:53:24,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:24,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:24,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:24,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:24,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T00:53:25,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:25,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T00:53:25,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372865080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:25,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:25,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372865195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:25,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:25,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T00:53:25,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:25,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:25,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52768 deadline: 1731372865236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:25,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:25,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372865236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:25,237 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 335e77c42adcd1ca9b481da0f68cba79 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:25,238 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 335e77c42adcd1ca9b481da0f68cba79 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:25,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/0a103485fd5e40c3a014c9963c4840c7 2024-11-12T00:53:25,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/9d10d39fb7fb45609e231947ecafc1f2 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/9d10d39fb7fb45609e231947ecafc1f2 2024-11-12T00:53:25,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/9d10d39fb7fb45609e231947ecafc1f2, entries=200, sequenceid=150, filesize=38.8 K 2024-11-12T00:53:25,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/18d9f6d7e41f4ad6930c56da166b19d4 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/18d9f6d7e41f4ad6930c56da166b19d4 2024-11-12T00:53:25,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/18d9f6d7e41f4ad6930c56da166b19d4, entries=100, sequenceid=150, filesize=9.5 K 2024-11-12T00:53:25,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/0a103485fd5e40c3a014c9963c4840c7 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/0a103485fd5e40c3a014c9963c4840c7 2024-11-12T00:53:25,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372865286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:25,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/0a103485fd5e40c3a014c9963c4840c7, entries=100, sequenceid=150, filesize=9.5 K 2024-11-12T00:53:25,294 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 335e77c42adcd1ca9b481da0f68cba79 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:25,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:25,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 335e77c42adcd1ca9b481da0f68cba79 in 683ms, sequenceid=150, compaction requested=true 2024-11-12T00:53:25,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:25,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:53:25,297 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:25,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:25,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:53:25,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:25,297 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:25,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:53:25,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:25,299 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46013 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:25,299 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/B is initiating minor compaction (all files) 2024-11-12T00:53:25,299 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/B in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,299 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/4789914765cf46daaa5c6ba84e0ecfa0, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/42c0b8c9fae64e728acefeb76fb3a3fb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/c311faf738e742c9a297534514e09c26, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/18d9f6d7e41f4ad6930c56da166b19d4] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=44.9 K 2024-11-12T00:53:25,299 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150047 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:25,299 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/A is initiating minor compaction (all files) 2024-11-12T00:53:25,299 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/A in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,300 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/0eca9cc4c18d41538456eb604ca260c4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/f6b0b36ba70a4270905bcec7602132d8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/6e5be26bb62149ec8bb2c95b94d7996e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/9d10d39fb7fb45609e231947ecafc1f2] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=146.5 K 2024-11-12T00:53:25,300 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,300 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/0eca9cc4c18d41538456eb604ca260c4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/f6b0b36ba70a4270905bcec7602132d8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/6e5be26bb62149ec8bb2c95b94d7996e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/9d10d39fb7fb45609e231947ecafc1f2] 2024-11-12T00:53:25,300 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 4789914765cf46daaa5c6ba84e0ecfa0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731372800797 2024-11-12T00:53:25,300 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eca9cc4c18d41538456eb604ca260c4, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731372800797 2024-11-12T00:53:25,301 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6b0b36ba70a4270905bcec7602132d8, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731372802726 2024-11-12T00:53:25,301 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 42c0b8c9fae64e728acefeb76fb3a3fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731372802729 2024-11-12T00:53:25,301 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e5be26bb62149ec8bb2c95b94d7996e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731372803112 2024-11-12T00:53:25,301 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting c311faf738e742c9a297534514e09c26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731372803112 2024-11-12T00:53:25,302 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d10d39fb7fb45609e231947ecafc1f2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731372804203 2024-11-12T00:53:25,302 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 18d9f6d7e41f4ad6930c56da166b19d4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731372804212 2024-11-12T00:53:25,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:25,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731372805014, max timestamp=1731372805299Num uniques 1018; 2024-11-12T00:53:25,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T00:53:25,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731372805014, max timestamp=1731372805299Num uniques 1018; 2024-11-12T00:53:25,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731372805014, max timestamp=1731372805299Num uniques 1018; 2024-11-12T00:53:25,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T00:53:25,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T00:53:25,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=40.25 KB heapSize=105.23 KB 2024-11-12T00:53:25,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:25,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:25,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:25,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:25,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:25,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:25,330 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#B#compaction#135 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:25,331 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/17a8104e162641fdb6817a45398c756f is 50, key is test_row_0/B:col10/1731372803122/Put/seqid=0 2024-11-12T00:53:25,338 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:25,358 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411127e35964bff7c4b9ea3bfddb45848ad6e_335e77c42adcd1ca9b481da0f68cba79 store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:25,361 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411127e35964bff7c4b9ea3bfddb45848ad6e_335e77c42adcd1ca9b481da0f68cba79, store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:25,361 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127e35964bff7c4b9ea3bfddb45848ad6e_335e77c42adcd1ca9b481da0f68cba79 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:25,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112877b0d245f1049a994a12c5469093e00_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372805314/Put/seqid=0 2024-11-12T00:53:25,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T00:53:25,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:25,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T00:53:25,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741982_1158 (size=12391) 2024-11-12T00:53:25,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741983_1159 (size=4469) 2024-11-12T00:53:25,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731372805388, max timestamp=1731372805417Num uniques 859; 2024-11-12T00:53:25,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T00:53:25,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731372805388, max timestamp=1731372805417Num uniques 859; 2024-11-12T00:53:25,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T00:53:25,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731372805388, max timestamp=1731372805417Num uniques 859; 2024-11-12T00:53:25,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T00:53:25,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741984_1160 (size=17284) 2024-11-12T00:53:25,438 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:25,439 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/17a8104e162641fdb6817a45398c756f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/17a8104e162641fdb6817a45398c756f 2024-11-12T00:53:25,443 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112877b0d245f1049a994a12c5469093e00_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112877b0d245f1049a994a12c5469093e00_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:25,444 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/f95028444b024b4b8309cf3ad4048e65, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:25,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/f95028444b024b4b8309cf3ad4048e65 is 175, key is test_row_0/A:col10/1731372805314/Put/seqid=0 2024-11-12T00:53:25,446 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/B of 335e77c42adcd1ca9b481da0f68cba79 into 17a8104e162641fdb6817a45398c756f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:25,446 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:25,446 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/B, priority=12, startTime=1731372805297; duration=0sec 2024-11-12T00:53:25,446 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:25,446 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:B 2024-11-12T00:53:25,446 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:25,448 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46013 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:25,448 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/C is initiating minor compaction (all files) 2024-11-12T00:53:25,448 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/C in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,449 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/3312e7eb3c9041669e15bc8b2a60b6dc, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/8388ca7b23e14c909c4ccfaa7631bc52, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/355be2668fcc4aaea7b57aa2d591cca3, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/0a103485fd5e40c3a014c9963c4840c7] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=44.9 K 2024-11-12T00:53:25,449 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 3312e7eb3c9041669e15bc8b2a60b6dc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731372800797 2024-11-12T00:53:25,450 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 8388ca7b23e14c909c4ccfaa7631bc52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731372802729 2024-11-12T00:53:25,450 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 355be2668fcc4aaea7b57aa2d591cca3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731372803112 2024-11-12T00:53:25,450 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a103485fd5e40c3a014c9963c4840c7, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731372804212 2024-11-12T00:53:25,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741985_1161 (size=48389) 2024-11-12T00:53:25,460 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#C#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:25,460 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/23fe35e20b214a5da3e0e2119840bb29 is 50, key is test_row_0/C:col10/1731372803122/Put/seqid=0 2024-11-12T00:53:25,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741986_1162 (size=12391) 2024-11-12T00:53:25,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:25,483 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/23fe35e20b214a5da3e0e2119840bb29 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/23fe35e20b214a5da3e0e2119840bb29 2024-11-12T00:53:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372865475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:25,488 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/C of 335e77c42adcd1ca9b481da0f68cba79 into 23fe35e20b214a5da3e0e2119840bb29(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:25,488 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:25,488 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/C, priority=12, startTime=1731372805297; duration=0sec 2024-11-12T00:53:25,489 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:25,489 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:C 2024-11-12T00:53:25,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372865482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:25,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T00:53:25,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:25,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:25,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372865595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372865596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:25,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T00:53:25,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:25,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372865806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:25,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372865806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:25,811 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#A#compaction#136 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:25,812 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/23b50a68644b457585ccb2506f7afe70 is 175, key is test_row_0/A:col10/1731372803122/Put/seqid=0 2024-11-12T00:53:25,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741987_1163 (size=31452) 2024-11-12T00:53:25,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T00:53:25,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:25,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,856 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/f95028444b024b4b8309cf3ad4048e65 2024-11-12T00:53:25,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/2a0fac81ac484b1d9bca74aa8c36cf0f is 50, key is test_row_0/B:col10/1731372805314/Put/seqid=0 2024-11-12T00:53:25,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741988_1164 (size=12151) 2024-11-12T00:53:25,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/2a0fac81ac484b1d9bca74aa8c36cf0f 2024-11-12T00:53:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T00:53:25,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/4940e463eb2344fe8e0f0dce2c689713 is 50, key is test_row_0/C:col10/1731372805314/Put/seqid=0 2024-11-12T00:53:25,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741989_1165 (size=12151) 2024-11-12T00:53:25,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T00:53:25,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:25,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:25,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:26,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372866115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:26,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:26,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372866115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:26,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:26,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T00:53:26,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:26,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:26,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:26,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:26,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:26,226 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/23b50a68644b457585ccb2506f7afe70 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/23b50a68644b457585ccb2506f7afe70 2024-11-12T00:53:26,234 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/A of 335e77c42adcd1ca9b481da0f68cba79 into 23b50a68644b457585ccb2506f7afe70(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:26,234 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:26,234 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/A, priority=12, startTime=1731372805297; duration=0sec 2024-11-12T00:53:26,234 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:26,234 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:A 2024-11-12T00:53:26,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T00:53:26,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:26,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:26,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:26,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:26,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:26,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/4940e463eb2344fe8e0f0dce2c689713 2024-11-12T00:53:26,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/f95028444b024b4b8309cf3ad4048e65 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/f95028444b024b4b8309cf3ad4048e65 2024-11-12T00:53:26,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/f95028444b024b4b8309cf3ad4048e65, entries=250, sequenceid=190, filesize=47.3 K 2024-11-12T00:53:26,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/2a0fac81ac484b1d9bca74aa8c36cf0f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/2a0fac81ac484b1d9bca74aa8c36cf0f 2024-11-12T00:53:26,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/2a0fac81ac484b1d9bca74aa8c36cf0f, entries=150, sequenceid=190, filesize=11.9 K 2024-11-12T00:53:26,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/4940e463eb2344fe8e0f0dce2c689713 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/4940e463eb2344fe8e0f0dce2c689713 2024-11-12T00:53:26,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/4940e463eb2344fe8e0f0dce2c689713, entries=150, sequenceid=190, filesize=11.9 K 2024-11-12T00:53:26,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~105.19 KB/107712, currentSize=147.60 KB/151140 for 335e77c42adcd1ca9b481da0f68cba79 in 1032ms, sequenceid=190, compaction requested=false 2024-11-12T00:53:26,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:26,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T00:53:26,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:26,448 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-12T00:53:26,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:26,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:26,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:26,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:26,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:26,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:26,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a4c7b149745a480f9320f0b1256f6115_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372805413/Put/seqid=0 2024-11-12T00:53:26,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741990_1166 (size=12304) 2024-11-12T00:53:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:26,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:26,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372866641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372866644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:26,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372866755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:26,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:26,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372866755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:26,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:26,866 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a4c7b149745a480f9320f0b1256f6115_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a4c7b149745a480f9320f0b1256f6115_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:26,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/05eeaaee4a5d4c92b66eb0e00c3fe55e, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:26,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/05eeaaee4a5d4c92b66eb0e00c3fe55e is 175, key is test_row_0/A:col10/1731372805413/Put/seqid=0 2024-11-12T00:53:26,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741991_1167 (size=31105) 2024-11-12T00:53:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T00:53:26,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372866965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:26,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:26,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372866965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:27,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372867255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:27,257 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 335e77c42adcd1ca9b481da0f68cba79 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:27,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52768 deadline: 1731372867266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:27,266 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 335e77c42adcd1ca9b481da0f68cba79 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:27,274 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/05eeaaee4a5d4c92b66eb0e00c3fe55e 2024-11-12T00:53:27,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372867276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:27,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372867276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:27,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:27,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:27,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:27,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/fad942599f524715b73d5b041f0f975a is 50, key is test_row_0/B:col10/1731372805413/Put/seqid=0 2024-11-12T00:53:27,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741992_1168 (size=12151) 2024-11-12T00:53:27,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372867315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:27,316 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 335e77c42adcd1ca9b481da0f68cba79 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:27,688 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/fad942599f524715b73d5b041f0f975a 2024-11-12T00:53:27,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/fb5f2f00b97142dba773807ec32107eb is 50, key is test_row_0/C:col10/1731372805413/Put/seqid=0 2024-11-12T00:53:27,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741993_1169 (size=12151) 2024-11-12T00:53:27,715 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/fb5f2f00b97142dba773807ec32107eb 2024-11-12T00:53:27,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/05eeaaee4a5d4c92b66eb0e00c3fe55e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/05eeaaee4a5d4c92b66eb0e00c3fe55e 2024-11-12T00:53:27,726 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/05eeaaee4a5d4c92b66eb0e00c3fe55e, entries=150, sequenceid=229, filesize=30.4 K 2024-11-12T00:53:27,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/fad942599f524715b73d5b041f0f975a as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/fad942599f524715b73d5b041f0f975a 2024-11-12T00:53:27,732 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/fad942599f524715b73d5b041f0f975a, entries=150, sequenceid=229, filesize=11.9 K 2024-11-12T00:53:27,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/fb5f2f00b97142dba773807ec32107eb as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/fb5f2f00b97142dba773807ec32107eb 2024-11-12T00:53:27,738 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/fb5f2f00b97142dba773807ec32107eb, entries=150, sequenceid=229, filesize=11.9 K 2024-11-12T00:53:27,739 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 335e77c42adcd1ca9b481da0f68cba79 in 1291ms, sequenceid=229, compaction requested=true 2024-11-12T00:53:27,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:27,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:27,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-12T00:53:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-12T00:53:27,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-12T00:53:27,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9870 sec 2024-11-12T00:53:27,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.9950 sec 2024-11-12T00:53:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:27,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T00:53:27,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:27,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:27,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:27,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:27,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:27,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:27,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122b12b8e196c44ba0939db48a74715e32_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372807786/Put/seqid=0 2024-11-12T00:53:27,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741994_1170 (size=14794) 2024-11-12T00:53:27,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731372807821, max timestamp=1731372807856Num uniques 859; 2024-11-12T00:53:27,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:53:27,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372807821, max timestamp=1731372807856Num uniques 859; 2024-11-12T00:53:27,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:53:27,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372807821, max timestamp=1731372807856Num uniques 859; 2024-11-12T00:53:27,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:53:27,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372867908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:27,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372867909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372868026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372868026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:28,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:28,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372868235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:28,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372868235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:28,251 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:28,254 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122b12b8e196c44ba0939db48a74715e32_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122b12b8e196c44ba0939db48a74715e32_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:28,255 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/feed6cb8bd6d4a8f98d0bf7de5b751bb, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:28,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/feed6cb8bd6d4a8f98d0bf7de5b751bb is 175, key is test_row_0/A:col10/1731372807786/Put/seqid=0 2024-11-12T00:53:28,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741995_1171 (size=39749) 2024-11-12T00:53:28,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372868545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:28,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372868545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:28,664 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/feed6cb8bd6d4a8f98d0bf7de5b751bb 2024-11-12T00:53:28,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/062f595b2b3d4bc9868b89b86a4411c8 is 50, key is test_row_0/B:col10/1731372807786/Put/seqid=0 2024-11-12T00:53:28,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741996_1172 (size=12151) 2024-11-12T00:53:28,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/062f595b2b3d4bc9868b89b86a4411c8 2024-11-12T00:53:28,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/8d7c183ba41f4725b9c65283bd8f2d80 is 50, key is test_row_0/C:col10/1731372807786/Put/seqid=0 2024-11-12T00:53:28,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741997_1173 (size=12151) 2024-11-12T00:53:28,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/8d7c183ba41f4725b9c65283bd8f2d80 2024-11-12T00:53:28,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/feed6cb8bd6d4a8f98d0bf7de5b751bb as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/feed6cb8bd6d4a8f98d0bf7de5b751bb 2024-11-12T00:53:28,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/feed6cb8bd6d4a8f98d0bf7de5b751bb, entries=200, sequenceid=239, filesize=38.8 K 2024-11-12T00:53:28,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/062f595b2b3d4bc9868b89b86a4411c8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/062f595b2b3d4bc9868b89b86a4411c8 2024-11-12T00:53:28,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/062f595b2b3d4bc9868b89b86a4411c8, entries=150, sequenceid=239, filesize=11.9 K 2024-11-12T00:53:28,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/8d7c183ba41f4725b9c65283bd8f2d80 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/8d7c183ba41f4725b9c65283bd8f2d80 2024-11-12T00:53:28,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/8d7c183ba41f4725b9c65283bd8f2d80, entries=150, sequenceid=239, filesize=11.9 K 2024-11-12T00:53:28,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 335e77c42adcd1ca9b481da0f68cba79 in 946ms, sequenceid=239, compaction requested=true 2024-11-12T00:53:28,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:28,734 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:28,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:53:28,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:28,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:53:28,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:28,735 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:28,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:53:28,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:28,741 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:28,741 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150695 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:28,741 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/B is initiating minor compaction (all files) 2024-11-12T00:53:28,741 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/A is initiating minor compaction (all files) 2024-11-12T00:53:28,741 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/B in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:28,741 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/A in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:28,741 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/23b50a68644b457585ccb2506f7afe70, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/f95028444b024b4b8309cf3ad4048e65, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/05eeaaee4a5d4c92b66eb0e00c3fe55e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/feed6cb8bd6d4a8f98d0bf7de5b751bb] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=147.2 K 2024-11-12T00:53:28,741 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/17a8104e162641fdb6817a45398c756f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/2a0fac81ac484b1d9bca74aa8c36cf0f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/fad942599f524715b73d5b041f0f975a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/062f595b2b3d4bc9868b89b86a4411c8] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=47.7 K 2024-11-12T00:53:28,741 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:28,741 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/23b50a68644b457585ccb2506f7afe70, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/f95028444b024b4b8309cf3ad4048e65, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/05eeaaee4a5d4c92b66eb0e00c3fe55e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/feed6cb8bd6d4a8f98d0bf7de5b751bb] 2024-11-12T00:53:28,745 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 17a8104e162641fdb6817a45398c756f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731372803122 2024-11-12T00:53:28,745 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23b50a68644b457585ccb2506f7afe70, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731372803122 2024-11-12T00:53:28,746 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a0fac81ac484b1d9bca74aa8c36cf0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731372805014 2024-11-12T00:53:28,746 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting f95028444b024b4b8309cf3ad4048e65, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731372805014 2024-11-12T00:53:28,746 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting fad942599f524715b73d5b041f0f975a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731372805413 2024-11-12T00:53:28,747 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05eeaaee4a5d4c92b66eb0e00c3fe55e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731372805413 2024-11-12T00:53:28,747 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 062f595b2b3d4bc9868b89b86a4411c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731372806630 2024-11-12T00:53:28,747 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting feed6cb8bd6d4a8f98d0bf7de5b751bb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731372806630 2024-11-12T00:53:28,778 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:28,779 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:28,780 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/e57431fa2fe84ff6a1150ac8fcb5e031 is 50, key is test_row_0/B:col10/1731372807786/Put/seqid=0 2024-11-12T00:53:28,786 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411127dce80c03a394bb886e0ce978751a6ff_335e77c42adcd1ca9b481da0f68cba79 store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:28,789 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411127dce80c03a394bb886e0ce978751a6ff_335e77c42adcd1ca9b481da0f68cba79, store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:28,789 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127dce80c03a394bb886e0ce978751a6ff_335e77c42adcd1ca9b481da0f68cba79 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:28,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741998_1174 (size=12527) 2024-11-12T00:53:28,805 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/e57431fa2fe84ff6a1150ac8fcb5e031 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/e57431fa2fe84ff6a1150ac8fcb5e031 2024-11-12T00:53:28,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741999_1175 (size=4469) 2024-11-12T00:53:28,812 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/B of 335e77c42adcd1ca9b481da0f68cba79 into e57431fa2fe84ff6a1150ac8fcb5e031(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:28,812 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:28,812 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/B, priority=12, startTime=1731372808735; duration=0sec 2024-11-12T00:53:28,813 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:28,813 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:B 2024-11-12T00:53:28,813 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:28,815 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:28,815 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/C is initiating minor compaction (all files) 2024-11-12T00:53:28,815 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/C in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:28,815 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/23fe35e20b214a5da3e0e2119840bb29, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/4940e463eb2344fe8e0f0dce2c689713, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/fb5f2f00b97142dba773807ec32107eb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/8d7c183ba41f4725b9c65283bd8f2d80] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=47.7 K 2024-11-12T00:53:28,816 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 23fe35e20b214a5da3e0e2119840bb29, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731372803122 2024-11-12T00:53:28,816 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 4940e463eb2344fe8e0f0dce2c689713, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731372805014 2024-11-12T00:53:28,816 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting fb5f2f00b97142dba773807ec32107eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731372805413 2024-11-12T00:53:28,817 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d7c183ba41f4725b9c65283bd8f2d80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731372806630 2024-11-12T00:53:28,831 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#C#compaction#149 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:28,831 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/01972d5e9d8d4c60bf4d15e2c5dcb14e is 50, key is test_row_0/C:col10/1731372807786/Put/seqid=0 2024-11-12T00:53:28,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742000_1176 (size=12527) 2024-11-12T00:53:28,882 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/01972d5e9d8d4c60bf4d15e2c5dcb14e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/01972d5e9d8d4c60bf4d15e2c5dcb14e 2024-11-12T00:53:28,892 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/C of 335e77c42adcd1ca9b481da0f68cba79 into 01972d5e9d8d4c60bf4d15e2c5dcb14e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:28,892 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:28,892 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/C, priority=12, startTime=1731372808735; duration=0sec 2024-11-12T00:53:28,892 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:28,892 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:C 2024-11-12T00:53:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T00:53:28,916 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:28,917 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-12T00:53:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T00:53:28,919 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:28,919 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:28,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T00:53:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:29,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=143.13 KB heapSize=410.69 KB 2024-11-12T00:53:29,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:29,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T00:53:29,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:29,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T00:53:29,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:29,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T00:53:29,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f1f3540092d14779b5f92dd58b357635_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372807907/Put/seqid=0 2024-11-12T00:53:29,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T00:53:29,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:29,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:29,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:29,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742001_1177 (size=14994) 2024-11-12T00:53:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,075 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372869076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372869076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:29,080 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f1f3540092d14779b5f92dd58b357635_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f1f3540092d14779b5f92dd58b357635_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:29,081 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/bf6dce4764d64e73bbaecab24ed8c62b, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:29,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/bf6dce4764d64e73bbaecab24ed8c62b is 175, key is test_row_0/A:col10/1731372807907/Put/seqid=0 2024-11-12T00:53:29,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742002_1178 (size=39949) 2024-11-12T00:53:29,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372869185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:29,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:29,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372869185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:29,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:29,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:29,213 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#A#compaction#148 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:29,214 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/1f740ff96f994933bc1ca3d6e6ee2c30 is 175, key is test_row_0/A:col10/1731372807786/Put/seqid=0 2024-11-12T00:53:29,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742003_1179 (size=31481) 2024-11-12T00:53:29,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T00:53:29,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:29,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:29,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:29,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,227 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/1f740ff96f994933bc1ca3d6e6ee2c30 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/1f740ff96f994933bc1ca3d6e6ee2c30 2024-11-12T00:53:29,233 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/A of 335e77c42adcd1ca9b481da0f68cba79 into 1f740ff96f994933bc1ca3d6e6ee2c30(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:29,233 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:29,233 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/A, priority=12, startTime=1731372808734; duration=0sec 2024-11-12T00:53:29,233 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:29,233 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:A 2024-11-12T00:53:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T00:53:29,297 INFO [master/6928b69dca1b:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-12T00:53:29,297 INFO [master/6928b69dca1b:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-12T00:53:29,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T00:53:29,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:29,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:29,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:29,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372869395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:29,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372869395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:29,486 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/bf6dce4764d64e73bbaecab24ed8c62b 2024-11-12T00:53:29,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/d674d21ea2874d208b3efd348fc49990 is 50, key is test_row_0/B:col10/1731372807907/Put/seqid=0 2024-11-12T00:53:29,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742004_1180 (size=12301) 2024-11-12T00:53:29,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/d674d21ea2874d208b3efd348fc49990 2024-11-12T00:53:29,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/21727f45d69a413886428bd7e56a474e is 50, key is test_row_0/C:col10/1731372807907/Put/seqid=0 2024-11-12T00:53:29,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742005_1181 (size=12301) 2024-11-12T00:53:29,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T00:53:29,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:29,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:29,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:29,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T00:53:29,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T00:53:29,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:29,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:29,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:29,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372869706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:29,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:29,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372869706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:29,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T00:53:29,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:29,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:29,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:29,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:29,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/21727f45d69a413886428bd7e56a474e 2024-11-12T00:53:29,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/bf6dce4764d64e73bbaecab24ed8c62b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/bf6dce4764d64e73bbaecab24ed8c62b 2024-11-12T00:53:29,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/bf6dce4764d64e73bbaecab24ed8c62b, entries=200, sequenceid=279, filesize=39.0 K 2024-11-12T00:53:29,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/d674d21ea2874d208b3efd348fc49990 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d674d21ea2874d208b3efd348fc49990 2024-11-12T00:53:29,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d674d21ea2874d208b3efd348fc49990, entries=150, sequenceid=279, filesize=12.0 K 2024-11-12T00:53:29,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/21727f45d69a413886428bd7e56a474e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/21727f45d69a413886428bd7e56a474e 2024-11-12T00:53:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/21727f45d69a413886428bd7e56a474e, entries=150, sequenceid=279, filesize=12.0 K 2024-11-12T00:53:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~410.64 KB/420496, currentSize=40.25 KB/41220 for 335e77c42adcd1ca9b481da0f68cba79 in 871ms, sequenceid=279, compaction requested=false 2024-11-12T00:53:29,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T00:53:29,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:29,991 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-12T00:53:29,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:29,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:29,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:29,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:29,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:29,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a150fe5bf50d436984706a0f221036ec_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372809073/Put/seqid=0 2024-11-12T00:53:30,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742006_1182 (size=9914) 2024-11-12T00:53:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,026 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a150fe5bf50d436984706a0f221036ec_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a150fe5bf50d436984706a0f221036ec_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/8165377ec81f4636bd86969897bf5872, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/8165377ec81f4636bd86969897bf5872 is 175, key is test_row_0/A:col10/1731372809073/Put/seqid=0 2024-11-12T00:53:30,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742007_1183 (size=22561) 2024-11-12T00:53:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,049 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/8165377ec81f4636bd86969897bf5872 2024-11-12T00:53:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T00:53:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/f3787d1a92ca4c69aa2f28155ab95361 is 50, key is test_row_0/B:col10/1731372809073/Put/seqid=0 2024-11-12T00:53:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742008_1184 (size=9857) 2024-11-12T00:53:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,077 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/f3787d1a92ca4c69aa2f28155ab95361 2024-11-12T00:53:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/9716ab32023c474583ec2f1f5a3bc740 is 50, key is test_row_0/C:col10/1731372809073/Put/seqid=0 2024-11-12T00:53:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742009_1185 (size=9857) 2024-11-12T00:53:30,103 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/9716ab32023c474583ec2f1f5a3bc740 2024-11-12T00:53:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/8165377ec81f4636bd86969897bf5872 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/8165377ec81f4636bd86969897bf5872 2024-11-12T00:53:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,117 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/8165377ec81f4636bd86969897bf5872, entries=100, sequenceid=289, filesize=22.0 K 2024-11-12T00:53:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/f3787d1a92ca4c69aa2f28155ab95361 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f3787d1a92ca4c69aa2f28155ab95361 2024-11-12T00:53:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,124 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f3787d1a92ca4c69aa2f28155ab95361, entries=100, sequenceid=289, filesize=9.6 K 2024-11-12T00:53:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/9716ab32023c474583ec2f1f5a3bc740 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/9716ab32023c474583ec2f1f5a3bc740 2024-11-12T00:53:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,133 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/9716ab32023c474583ec2f1f5a3bc740, entries=100, sequenceid=289, filesize=9.6 K 2024-11-12T00:53:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,136 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 335e77c42adcd1ca9b481da0f68cba79 in 145ms, sequenceid=289, compaction requested=true 2024-11-12T00:53:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-12T00:53:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-12T00:53:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-12T00:53:30,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2210 sec 2024-11-12T00:53:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,145 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.2260 sec 2024-11-12T00:53:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:30,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T00:53:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:30,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:30,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:30,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:30,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:30,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111244aa872e1ee749e6907f5b4fffd332b2_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372810298/Put/seqid=0 2024-11-12T00:53:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742010_1186 (size=17534) 2024-11-12T00:53:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731372810358, max timestamp=1731372810386Num uniques 859; 2024-11-12T00:53:30,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:53:30,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731372810358, max timestamp=1731372810386Num uniques 859; 2024-11-12T00:53:30,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-12T00:53:30,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731372810358, max timestamp=1731372810386Num uniques 859; 2024-11-12T00:53:30,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-12T00:53:30,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:30,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372870430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:30,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372870436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:30,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:30,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:30,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:30,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372870545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:30,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372870545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:30,751 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:30,756 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111244aa872e1ee749e6907f5b4fffd332b2_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111244aa872e1ee749e6907f5b4fffd332b2_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:30,757 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/42c5f6cbdff345ac91191c7c3531e711, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:30,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/42c5f6cbdff345ac91191c7c3531e711 is 175, key is test_row_0/A:col10/1731372810298/Put/seqid=0 2024-11-12T00:53:30,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372870755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:30,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372870755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:30,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742011_1187 (size=48635) 2024-11-12T00:53:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:30,761 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/42c5f6cbdff345ac91191c7c3531e711 2024-11-12T00:53:30,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/879f69cd9cfa4533b08507160c8aa53e is 50, key is test_row_0/B:col10/1731372810298/Put/seqid=0 2024-11-12T00:53:30,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742012_1188 (size=9857) 2024-11-12T00:53:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T00:53:31,066 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:31,066 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-12T00:53:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T00:53:31,068 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:31,070 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:31,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:31,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372871066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:31,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372871066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:31,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/879f69cd9cfa4533b08507160c8aa53e 2024-11-12T00:53:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T00:53:31,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/ffc439bc197d433b920bc2158d595683 is 50, key is test_row_0/C:col10/1731372810298/Put/seqid=0 2024-11-12T00:53:31,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742013_1189 (size=9857) 2024-11-12T00:53:31,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T00:53:31,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:31,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:31,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:31,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:31,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:31,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:31,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372871296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:31,297 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 335e77c42adcd1ca9b481da0f68cba79 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:31,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52768 deadline: 1731372871305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:31,311 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 335e77c42adcd1ca9b481da0f68cba79 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:31,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:31,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:31,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:31,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372871346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:31,350 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 335e77c42adcd1ca9b481da0f68cba79 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:31,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:31,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:31,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T00:53:31,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:31,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:31,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:31,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:31,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T00:53:31,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T00:53:31,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:31,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:31,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:31,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:31,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:31,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372871575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:31,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/ffc439bc197d433b920bc2158d595683 2024-11-12T00:53:31,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/42c5f6cbdff345ac91191c7c3531e711 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/42c5f6cbdff345ac91191c7c3531e711 2024-11-12T00:53:31,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372871586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:31,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:31,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:31,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:31,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/42c5f6cbdff345ac91191c7c3531e711, entries=250, sequenceid=299, filesize=47.5 K 2024-11-12T00:53:31,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/879f69cd9cfa4533b08507160c8aa53e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/879f69cd9cfa4533b08507160c8aa53e 2024-11-12T00:53:31,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/879f69cd9cfa4533b08507160c8aa53e, entries=100, sequenceid=299, filesize=9.6 K 2024-11-12T00:53:31,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/ffc439bc197d433b920bc2158d595683 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/ffc439bc197d433b920bc2158d595683 2024-11-12T00:53:31,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/ffc439bc197d433b920bc2158d595683, entries=100, sequenceid=299, filesize=9.6 K 2024-11-12T00:53:31,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=135.63 KB/138882 for 335e77c42adcd1ca9b481da0f68cba79 in 1301ms, sequenceid=299, compaction requested=true 2024-11-12T00:53:31,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:31,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:53:31,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:31,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:53:31,611 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:31,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:31,611 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:31,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:53:31,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:31,612 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44542 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:31,612 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142626 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:31,612 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/A is initiating minor compaction (all files) 2024-11-12T00:53:31,612 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/B is initiating minor compaction (all files) 2024-11-12T00:53:31,613 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/A in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:31,613 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/B in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:31,613 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/e57431fa2fe84ff6a1150ac8fcb5e031, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d674d21ea2874d208b3efd348fc49990, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f3787d1a92ca4c69aa2f28155ab95361, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/879f69cd9cfa4533b08507160c8aa53e] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=43.5 K 2024-11-12T00:53:31,613 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/1f740ff96f994933bc1ca3d6e6ee2c30, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/bf6dce4764d64e73bbaecab24ed8c62b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/8165377ec81f4636bd86969897bf5872, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/42c5f6cbdff345ac91191c7c3531e711] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=139.3 K 2024-11-12T00:53:31,613 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:31,613 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/1f740ff96f994933bc1ca3d6e6ee2c30, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/bf6dce4764d64e73bbaecab24ed8c62b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/8165377ec81f4636bd86969897bf5872, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/42c5f6cbdff345ac91191c7c3531e711] 2024-11-12T00:53:31,613 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting e57431fa2fe84ff6a1150ac8fcb5e031, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731372806630 2024-11-12T00:53:31,613 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f740ff96f994933bc1ca3d6e6ee2c30, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731372806630 2024-11-12T00:53:31,614 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting d674d21ea2874d208b3efd348fc49990, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731372807905 2024-11-12T00:53:31,614 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf6dce4764d64e73bbaecab24ed8c62b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731372807902 2024-11-12T00:53:31,614 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting f3787d1a92ca4c69aa2f28155ab95361, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731372809073 2024-11-12T00:53:31,614 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8165377ec81f4636bd86969897bf5872, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731372809073 2024-11-12T00:53:31,614 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 879f69cd9cfa4533b08507160c8aa53e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731372810298 2024-11-12T00:53:31,614 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42c5f6cbdff345ac91191c7c3531e711, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731372810279 2024-11-12T00:53:31,624 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:31,627 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411121bf27fb8fe1b42d88025b165b981ddd0_335e77c42adcd1ca9b481da0f68cba79 store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:31,628 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:31,628 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/90b37b05d0d44535a4398d21418a2e9c is 50, key is test_row_0/B:col10/1731372810298/Put/seqid=0 2024-11-12T00:53:31,629 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411121bf27fb8fe1b42d88025b165b981ddd0_335e77c42adcd1ca9b481da0f68cba79, store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:31,629 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121bf27fb8fe1b42d88025b165b981ddd0_335e77c42adcd1ca9b481da0f68cba79 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:31,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742014_1190 (size=12813) 2024-11-12T00:53:31,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742015_1191 (size=4469) 2024-11-12T00:53:31,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T00:53:31,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:31,681 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=135.63 KB heapSize=388.23 KB 2024-11-12T00:53:31,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:31,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:31,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:31,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:31,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:31,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:31,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fb66a25415a54e88ada1d639ff96fc9c_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372810435/Put/seqid=0 2024-11-12T00:53:31,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742016_1192 (size=12454) 2024-11-12T00:53:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T00:53:31,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:31,706 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fb66a25415a54e88ada1d639ff96fc9c_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112fb66a25415a54e88ada1d639ff96fc9c_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:31,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/3d10242327e648e399cf1f309c477a59, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:31,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/3d10242327e648e399cf1f309c477a59 is 175, key is test_row_0/A:col10/1731372810435/Put/seqid=0 2024-11-12T00:53:31,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742017_1193 (size=31255) 2024-11-12T00:53:32,039 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#A#compaction#159 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:32,040 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/90b37b05d0d44535a4398d21418a2e9c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/90b37b05d0d44535a4398d21418a2e9c 2024-11-12T00:53:32,040 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/de4213cd2fef415c8e1fbc74d2741c71 is 175, key is test_row_0/A:col10/1731372810298/Put/seqid=0 2024-11-12T00:53:32,045 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/B of 335e77c42adcd1ca9b481da0f68cba79 into 90b37b05d0d44535a4398d21418a2e9c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:32,045 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:32,045 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/B, priority=12, startTime=1731372811611; duration=0sec 2024-11-12T00:53:32,045 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:32,045 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:B 2024-11-12T00:53:32,046 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:32,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742018_1194 (size=31874) 2024-11-12T00:53:32,047 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44542 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:32,047 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/C is initiating minor compaction (all files) 2024-11-12T00:53:32,047 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/C in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:32,047 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/01972d5e9d8d4c60bf4d15e2c5dcb14e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/21727f45d69a413886428bd7e56a474e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/9716ab32023c474583ec2f1f5a3bc740, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/ffc439bc197d433b920bc2158d595683] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=43.5 K 2024-11-12T00:53:32,048 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 01972d5e9d8d4c60bf4d15e2c5dcb14e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731372806630 2024-11-12T00:53:32,048 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 21727f45d69a413886428bd7e56a474e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731372807905 2024-11-12T00:53:32,048 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 9716ab32023c474583ec2f1f5a3bc740, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731372809073 2024-11-12T00:53:32,049 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting ffc439bc197d433b920bc2158d595683, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731372810298 2024-11-12T00:53:32,051 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/de4213cd2fef415c8e1fbc74d2741c71 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/de4213cd2fef415c8e1fbc74d2741c71 2024-11-12T00:53:32,059 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#C#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:32,060 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/77d6deb05ef24da28123501dbf331faf is 50, key is test_row_0/C:col10/1731372810298/Put/seqid=0 2024-11-12T00:53:32,066 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/A of 335e77c42adcd1ca9b481da0f68cba79 into de4213cd2fef415c8e1fbc74d2741c71(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:32,066 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:32,066 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/A, priority=12, startTime=1731372811611; duration=0sec 2024-11-12T00:53:32,066 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:32,066 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:A 2024-11-12T00:53:32,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742019_1195 (size=12813) 2024-11-12T00:53:32,077 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/77d6deb05ef24da28123501dbf331faf as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/77d6deb05ef24da28123501dbf331faf 2024-11-12T00:53:32,082 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/C of 335e77c42adcd1ca9b481da0f68cba79 into 77d6deb05ef24da28123501dbf331faf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:32,083 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:32,083 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/C, priority=12, startTime=1731372811611; duration=0sec 2024-11-12T00:53:32,083 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:32,083 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:C 2024-11-12T00:53:32,122 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/3d10242327e648e399cf1f309c477a59 2024-11-12T00:53:32,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/f0accbeb10984392b5fcc9111f14d7d4 is 50, key is test_row_0/B:col10/1731372810435/Put/seqid=0 2024-11-12T00:53:32,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742020_1196 (size=12301) 2024-11-12T00:53:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T00:53:32,545 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/f0accbeb10984392b5fcc9111f14d7d4 2024-11-12T00:53:32,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/09f28b34d28047808d1be4a660a31d76 is 50, key is test_row_0/C:col10/1731372810435/Put/seqid=0 2024-11-12T00:53:32,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742021_1197 (size=12301) 2024-11-12T00:53:32,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:32,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:32,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:32,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372872610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:32,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:32,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:32,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:32,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372872615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:32,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372872726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:32,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:32,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:32,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:32,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372872726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:32,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:32,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:32,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:32,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372872936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:32,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372872936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:32,959 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/09f28b34d28047808d1be4a660a31d76 2024-11-12T00:53:32,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/3d10242327e648e399cf1f309c477a59 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/3d10242327e648e399cf1f309c477a59 2024-11-12T00:53:32,975 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/3d10242327e648e399cf1f309c477a59, entries=150, sequenceid=336, filesize=30.5 K 2024-11-12T00:53:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/f0accbeb10984392b5fcc9111f14d7d4 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f0accbeb10984392b5fcc9111f14d7d4 2024-11-12T00:53:32,982 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f0accbeb10984392b5fcc9111f14d7d4, entries=150, sequenceid=336, filesize=12.0 K 2024-11-12T00:53:32,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/09f28b34d28047808d1be4a660a31d76 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/09f28b34d28047808d1be4a660a31d76 2024-11-12T00:53:32,986 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/09f28b34d28047808d1be4a660a31d76, entries=150, sequenceid=336, filesize=12.0 K 2024-11-12T00:53:32,987 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~135.63 KB/138882, heapSize ~388.19 KB/397504, currentSize=46.96 KB/48090 for 335e77c42adcd1ca9b481da0f68cba79 in 1306ms, sequenceid=336, compaction requested=false 2024-11-12T00:53:32,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:32,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:32,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-12T00:53:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-12T00:53:32,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-12T00:53:32,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9190 sec 2024-11-12T00:53:32,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.9250 sec 2024-11-12T00:53:33,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T00:53:33,226 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:33,226 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:33,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-12T00:53:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T00:53:33,228 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:33,229 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:33,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:33,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T00:53:33,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:33,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:33,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:33,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:33,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:33,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:33,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112889dc6e7e07e41d6b176b25359cd68e4_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372813246/Put/seqid=0 2024-11-12T00:53:33,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742022_1198 (size=17534) 2024-11-12T00:53:33,283 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:33,288 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112889dc6e7e07e41d6b176b25359cd68e4_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112889dc6e7e07e41d6b176b25359cd68e4_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:33,290 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/908d0d764ace41678c9241c7ba626a4e, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:33,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/908d0d764ace41678c9241c7ba626a4e is 175, key is test_row_0/A:col10/1731372813246/Put/seqid=0 2024-11-12T00:53:33,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742023_1199 (size=48639) 2024-11-12T00:53:33,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:33,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372873320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:33,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:33,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:33,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372873321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:33,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T00:53:33,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T00:53:33,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:33,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:33,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:33,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:33,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:33,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:33,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:33,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372873426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:33,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372873436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:33,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T00:53:33,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:33,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:33,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:33,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:33,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T00:53:33,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372873635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:33,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:33,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372873646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:33,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T00:53:33,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:33,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:33,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:33,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:33,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:33,699 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/908d0d764ace41678c9241c7ba626a4e 2024-11-12T00:53:33,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/37f112df6fe84f18b9a838db277afbd0 is 50, key is test_row_0/B:col10/1731372813246/Put/seqid=0 2024-11-12T00:53:33,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742024_1200 (size=12301) 2024-11-12T00:53:33,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T00:53:33,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:33,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:33,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:33,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:33,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T00:53:33,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372873946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:33,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372873955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:33,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T00:53:33,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:33,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:33,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:33,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:33,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:34,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/37f112df6fe84f18b9a838db277afbd0 2024-11-12T00:53:34,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/af705477c8f64d14972e40203a1b016d is 50, key is test_row_0/C:col10/1731372813246/Put/seqid=0 2024-11-12T00:53:34,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742025_1201 (size=12301) 2024-11-12T00:53:34,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T00:53:34,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:34,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:34,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:34,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:34,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:34,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T00:53:34,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:34,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:34,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:34,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:34,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:34,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T00:53:34,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T00:53:34,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:34,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:34,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:34,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:34,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:34,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372874455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:34,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372874468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:34,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:34,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:34,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:34,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/af705477c8f64d14972e40203a1b016d 2024-11-12T00:53:34,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/908d0d764ace41678c9241c7ba626a4e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/908d0d764ace41678c9241c7ba626a4e 2024-11-12T00:53:34,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/908d0d764ace41678c9241c7ba626a4e, entries=250, sequenceid=351, filesize=47.5 K 2024-11-12T00:53:34,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/37f112df6fe84f18b9a838db277afbd0 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/37f112df6fe84f18b9a838db277afbd0 2024-11-12T00:53:34,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/37f112df6fe84f18b9a838db277afbd0, entries=150, sequenceid=351, filesize=12.0 K 2024-11-12T00:53:34,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/af705477c8f64d14972e40203a1b016d as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/af705477c8f64d14972e40203a1b016d 2024-11-12T00:53:34,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/af705477c8f64d14972e40203a1b016d, entries=150, sequenceid=351, filesize=12.0 K 2024-11-12T00:53:34,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 335e77c42adcd1ca9b481da0f68cba79 in 1274ms, sequenceid=351, compaction requested=true 2024-11-12T00:53:34,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:34,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:53:34,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:34,542 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:34,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:53:34,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:34,542 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:34,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:53:34,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:34,543 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:34,543 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111768 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:34,543 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/B is initiating minor compaction (all files) 2024-11-12T00:53:34,543 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/A is initiating minor compaction (all files) 2024-11-12T00:53:34,543 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/A in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:34,543 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/B in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:34,543 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/de4213cd2fef415c8e1fbc74d2741c71, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/3d10242327e648e399cf1f309c477a59, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/908d0d764ace41678c9241c7ba626a4e] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=109.1 K 2024-11-12T00:53:34,543 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/90b37b05d0d44535a4398d21418a2e9c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f0accbeb10984392b5fcc9111f14d7d4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/37f112df6fe84f18b9a838db277afbd0] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=36.5 K 2024-11-12T00:53:34,544 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:34,544 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/de4213cd2fef415c8e1fbc74d2741c71, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/3d10242327e648e399cf1f309c477a59, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/908d0d764ace41678c9241c7ba626a4e] 2024-11-12T00:53:34,544 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 90b37b05d0d44535a4398d21418a2e9c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731372809057 2024-11-12T00:53:34,544 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting de4213cd2fef415c8e1fbc74d2741c71, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731372809057 2024-11-12T00:53:34,544 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting f0accbeb10984392b5fcc9111f14d7d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731372810428 2024-11-12T00:53:34,544 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d10242327e648e399cf1f309c477a59, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731372810428 2024-11-12T00:53:34,544 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 37f112df6fe84f18b9a838db277afbd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731372812608 2024-11-12T00:53:34,545 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 908d0d764ace41678c9241c7ba626a4e, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731372812601 2024-11-12T00:53:34,550 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:34,551 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:34,552 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/a85db80d19414bf8bd5f9045891f2705 is 50, key is test_row_0/B:col10/1731372813246/Put/seqid=0 2024-11-12T00:53:34,553 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112730f598d02bc4b07aec73bae89de95bc_335e77c42adcd1ca9b481da0f68cba79 store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:34,555 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112730f598d02bc4b07aec73bae89de95bc_335e77c42adcd1ca9b481da0f68cba79, store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:34,555 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112730f598d02bc4b07aec73bae89de95bc_335e77c42adcd1ca9b481da0f68cba79 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:34,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742026_1202 (size=12915) 2024-11-12T00:53:34,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742027_1203 (size=4469) 2024-11-12T00:53:34,562 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#A#compaction#169 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:34,563 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/bb95059292c449cfb29a008de0350c93 is 175, key is test_row_0/A:col10/1731372813246/Put/seqid=0 2024-11-12T00:53:34,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742028_1204 (size=31869) 2024-11-12T00:53:34,573 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/bb95059292c449cfb29a008de0350c93 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/bb95059292c449cfb29a008de0350c93 2024-11-12T00:53:34,579 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/A of 335e77c42adcd1ca9b481da0f68cba79 into bb95059292c449cfb29a008de0350c93(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:34,579 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:34,579 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/A, priority=13, startTime=1731372814542; duration=0sec 2024-11-12T00:53:34,579 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:34,579 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:A 2024-11-12T00:53:34,579 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:34,580 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:34,580 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/C is initiating minor compaction (all files) 2024-11-12T00:53:34,580 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/C in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:34,580 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/77d6deb05ef24da28123501dbf331faf, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/09f28b34d28047808d1be4a660a31d76, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/af705477c8f64d14972e40203a1b016d] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=36.5 K 2024-11-12T00:53:34,581 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77d6deb05ef24da28123501dbf331faf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731372809057 2024-11-12T00:53:34,581 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09f28b34d28047808d1be4a660a31d76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731372810428 2024-11-12T00:53:34,581 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting af705477c8f64d14972e40203a1b016d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731372812608 2024-11-12T00:53:34,592 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:34,592 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/29ee7296c6a34eb6867284a7ac2017be is 50, key is test_row_0/C:col10/1731372813246/Put/seqid=0 2024-11-12T00:53:34,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742029_1205 (size=12915) 2024-11-12T00:53:34,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T00:53:34,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:34,608 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T00:53:34,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:34,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:34,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:34,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:34,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:34,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:34,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111294187c33a9eb4a86bee5b76115ecc283_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372813319/Put/seqid=0 2024-11-12T00:53:34,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742030_1206 (size=12454) 2024-11-12T00:53:34,964 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/a85db80d19414bf8bd5f9045891f2705 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/a85db80d19414bf8bd5f9045891f2705 2024-11-12T00:53:34,970 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/B of 335e77c42adcd1ca9b481da0f68cba79 into a85db80d19414bf8bd5f9045891f2705(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:34,970 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:34,970 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/B, priority=13, startTime=1731372814542; duration=0sec 2024-11-12T00:53:34,970 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:34,970 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:B 2024-11-12T00:53:35,002 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/29ee7296c6a34eb6867284a7ac2017be as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/29ee7296c6a34eb6867284a7ac2017be 2024-11-12T00:53:35,009 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/C of 335e77c42adcd1ca9b481da0f68cba79 into 29ee7296c6a34eb6867284a7ac2017be(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:35,009 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:35,009 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/C, priority=13, startTime=1731372814542; duration=0sec 2024-11-12T00:53:35,009 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:35,010 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:C 2024-11-12T00:53:35,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,034 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111294187c33a9eb4a86bee5b76115ecc283_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111294187c33a9eb4a86bee5b76115ecc283_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:35,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/1b5a994d719545bfbaf57890a5c2ac55, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:35,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/1b5a994d719545bfbaf57890a5c2ac55 is 175, key is test_row_0/A:col10/1731372813319/Put/seqid=0 2024-11-12T00:53:35,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742031_1207 (size=31255) 2024-11-12T00:53:35,040 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/1b5a994d719545bfbaf57890a5c2ac55 2024-11-12T00:53:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/674ca31cee9e416abf74962cefda3016 is 50, key is test_row_0/B:col10/1731372813319/Put/seqid=0 2024-11-12T00:53:35,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742032_1208 (size=12301) 2024-11-12T00:53:35,059 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/674ca31cee9e416abf74962cefda3016 2024-11-12T00:53:35,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/bf20b6fc53694478bb443e2ec2d0f398 is 50, key is test_row_0/C:col10/1731372813319/Put/seqid=0 2024-11-12T00:53:35,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742033_1209 (size=12301) 2024-11-12T00:53:35,085 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/bf20b6fc53694478bb443e2ec2d0f398 2024-11-12T00:53:35,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/1b5a994d719545bfbaf57890a5c2ac55 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/1b5a994d719545bfbaf57890a5c2ac55 2024-11-12T00:53:35,094 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/1b5a994d719545bfbaf57890a5c2ac55, entries=150, sequenceid=372, filesize=30.5 K 2024-11-12T00:53:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/674ca31cee9e416abf74962cefda3016 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/674ca31cee9e416abf74962cefda3016 2024-11-12T00:53:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,100 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/674ca31cee9e416abf74962cefda3016, entries=150, sequenceid=372, filesize=12.0 K 2024-11-12T00:53:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/bf20b6fc53694478bb443e2ec2d0f398 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/bf20b6fc53694478bb443e2ec2d0f398 2024-11-12T00:53:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,108 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/bf20b6fc53694478bb443e2ec2d0f398, entries=150, sequenceid=372, filesize=12.0 K 2024-11-12T00:53:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,110 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 335e77c42adcd1ca9b481da0f68cba79 in 501ms, sequenceid=372, compaction requested=false 2024-11-12T00:53:35,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:35,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:35,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-12T00:53:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-12T00:53:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-12T00:53:35,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8820 sec 2024-11-12T00:53:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.8890 sec 2024-11-12T00:53:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T00:53:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,377 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,378 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-12T00:53:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T00:53:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,380 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,381 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:35,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T00:53:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T00:53:35,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,534 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T00:53:35,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:35,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:35,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:35,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:35,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:35,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111214a8bee817e04f90a000be3107ad69b8_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_1/A:col10/1731372815529/Put/seqid=0 2024-11-12T00:53:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742034_1210 (size=14994) 2024-11-12T00:53:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:35,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372815611, max timestamp=1731372815664Num uniques 859; 2024-11-12T00:53:35,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:53:35,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372815611, max timestamp=1731372815664Num uniques 859; 2024-11-12T00:53:35,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:53:35,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372815611, max timestamp=1731372815664Num uniques 859; 2024-11-12T00:53:35,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:53:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T00:53:35,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731372815708, max timestamp=1731372815736Num uniques 1018; 2024-11-12T00:53:35,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731372815708, max timestamp=1731372815736Num uniques 1018; 2024-11-12T00:53:35,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-12T00:53:35,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-12T00:53:35,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731372815708, max timestamp=1731372815736Num uniques 1018; 2024-11-12T00:53:35,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-12T00:53:35,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372875788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:35,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372875788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:35,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:35,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:35,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:35,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:35,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:35,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:35,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731372815759, max timestamp=1731372815786Num uniques 1177; 2024-11-12T00:53:35,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-12T00:53:35,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731372815759, max timestamp=1731372815786Num uniques 1177; 2024-11-12T00:53:35,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-12T00:53:35,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731372815769, max timestamp=1731372815786Num uniques 1177; 2024-11-12T00:53:35,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=6870, suffix heap size=150376,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=150 2024-11-12T00:53:35,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731372815921, max timestamp=1731372815934Num uniques 1136; 2024-11-12T00:53:35,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731372815921, max timestamp=1731372815934Num uniques 1136; 2024-11-12T00:53:35,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-12T00:53:35,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-12T00:53:35,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731372815921, max timestamp=1731372815934Num uniques 1009; 2024-11-12T00:53:35,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8516, suffix heap size=132904,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=186 2024-11-12T00:53:35,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731372815972, max timestamp=1731372815980Num uniques 1045; 2024-11-12T00:53:35,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731372815972, max timestamp=1731372815980Num uniques 1045; 2024-11-12T00:53:35,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-12T00:53:35,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-12T00:53:35,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731372815972, max timestamp=1731372815980Num uniques 1045; 2024-11-12T00:53:35,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-12T00:53:35,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:36,000 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111214a8bee817e04f90a000be3107ad69b8_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111214a8bee817e04f90a000be3107ad69b8_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:36,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/e72fdc94c2894289b51e12d1839e5817, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:36,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/e72fdc94c2894289b51e12d1839e5817 is 175, key is test_row_1/A:col10/1731372815529/Put/seqid=0 2024-11-12T00:53:36,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742035_1211 (size=39945) 2024-11-12T00:53:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T00:53:36,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1731372815999, max timestamp=1731372816037Num uniques 1054; 2024-11-12T00:53:36,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=13920, suffix heap size=137584,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=304 2024-11-12T00:53:36,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1731372815999, max timestamp=1731372816037Num uniques 1054; 2024-11-12T00:53:36,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1731372815999, max timestamp=1731372816037Num uniques 1054; 2024-11-12T00:53:36,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=13920, suffix heap size=137584,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=304 2024-11-12T00:53:36,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=13920, suffix heap size=137584,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=304 2024-11-12T00:53:36,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372876087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:36,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731372816075, max timestamp=1731372816087Num uniques 1163; 2024-11-12T00:53:36,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=7458, suffix heap size=148920,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=163 2024-11-12T00:53:36,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731372816075, max timestamp=1731372816087Num uniques 1163; 2024-11-12T00:53:36,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=7458, suffix heap size=148920,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=163 2024-11-12T00:53:36,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731372816075, max timestamp=1731372816087Num uniques 1163; 2024-11-12T00:53:36,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=7458, suffix heap size=148920,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=163 2024-11-12T00:53:36,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731372816136, max timestamp=1731372816191Num uniques 1022; 2024-11-12T00:53:36,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-12T00:53:36,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731372816136, max timestamp=1731372816191Num uniques 1022; 2024-11-12T00:53:36,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731372816136, max timestamp=1731372816191Num uniques 1022; 2024-11-12T00:53:36,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-12T00:53:36,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-12T00:53:36,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731372816225, max timestamp=1731372816288Num uniques 1081; 2024-11-12T00:53:36,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731372816225, max timestamp=1731372816288Num uniques 1081; 2024-11-12T00:53:36,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-12T00:53:36,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-12T00:53:36,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731372816225, max timestamp=1731372816288Num uniques 1081; 2024-11-12T00:53:36,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-12T00:53:36,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731372816334, max timestamp=1731372816380Num uniques 1040; 2024-11-12T00:53:36,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731372816334, max timestamp=1731372816380Num uniques 1040; 2024-11-12T00:53:36,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-12T00:53:36,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-12T00:53:36,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731372816334, max timestamp=1731372816380Num uniques 1040; 2024-11-12T00:53:36,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-12T00:53:36,406 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=381, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/e72fdc94c2894289b51e12d1839e5817 2024-11-12T00:53:36,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/ff6c95ecb0e84eb29e1488e19430b8b0 is 50, key is test_row_1/B:col10/1731372815529/Put/seqid=0 2024-11-12T00:53:36,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742036_1212 (size=9857) 2024-11-12T00:53:36,445 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/ff6c95ecb0e84eb29e1488e19430b8b0 2024-11-12T00:53:36,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731372816420, max timestamp=1731372816450Num uniques 1099; 2024-11-12T00:53:36,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731372816420, max timestamp=1731372816450Num uniques 1099; 2024-11-12T00:53:36,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-11-12T00:53:36,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-11-12T00:53:36,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731372816420, max timestamp=1731372816450Num uniques 1099; 2024-11-12T00:53:36,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-11-12T00:53:36,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/f4d5c23cdc9f45fb920ae26bc0407902 is 50, key is test_row_1/C:col10/1731372815529/Put/seqid=0 2024-11-12T00:53:36,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742037_1213 (size=9857) 2024-11-12T00:53:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T00:53:36,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372876513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:36,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:36,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372876518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:36,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:36,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:36,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:36,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372876626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:36,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:36,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:36,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:36,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372876636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:36,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372876835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:36,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:36,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:36,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:36,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372876846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:36,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:36,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:36,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:36,867 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/f4d5c23cdc9f45fb920ae26bc0407902 2024-11-12T00:53:36,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/e72fdc94c2894289b51e12d1839e5817 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/e72fdc94c2894289b51e12d1839e5817 2024-11-12T00:53:36,875 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/e72fdc94c2894289b51e12d1839e5817, entries=200, sequenceid=381, filesize=39.0 K 2024-11-12T00:53:36,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/ff6c95ecb0e84eb29e1488e19430b8b0 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/ff6c95ecb0e84eb29e1488e19430b8b0 2024-11-12T00:53:36,880 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/ff6c95ecb0e84eb29e1488e19430b8b0, entries=100, sequenceid=381, filesize=9.6 K 2024-11-12T00:53:36,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/f4d5c23cdc9f45fb920ae26bc0407902 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/f4d5c23cdc9f45fb920ae26bc0407902 2024-11-12T00:53:36,885 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/f4d5c23cdc9f45fb920ae26bc0407902, entries=100, sequenceid=381, filesize=9.6 K 2024-11-12T00:53:36,886 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=154.31 KB/158010 for 335e77c42adcd1ca9b481da0f68cba79 in 1352ms, sequenceid=381, compaction requested=true 2024-11-12T00:53:36,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:36,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:36,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-12T00:53:36,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-12T00:53:36,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-12T00:53:36,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5080 sec 2024-11-12T00:53:36,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.5130 sec 2024-11-12T00:53:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:37,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=161.02 KB heapSize=452.25 KB 2024-11-12T00:53:37,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:37,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T00:53:37,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:37,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T00:53:37,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:37,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T00:53:37,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a4fdac4e6de74d57b73e1e8f4ed84ce1_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372817147/Put/seqid=0 2024-11-12T00:53:37,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372877161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372877161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:37,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742038_1214 (size=14994) 2024-11-12T00:53:37,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372877276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:37,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372877276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:37,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372877485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:37,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372877485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T00:53:37,536 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:37,536 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-12T00:53:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T00:53:37,538 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:37,539 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:37,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:37,568 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:37,572 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a4fdac4e6de74d57b73e1e8f4ed84ce1_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a4fdac4e6de74d57b73e1e8f4ed84ce1_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:37,573 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/c62b688e8540434f9cbba89d1606db71, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:37,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/c62b688e8540434f9cbba89d1606db71 is 175, key is test_row_0/A:col10/1731372817147/Put/seqid=0 2024-11-12T00:53:37,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742039_1215 (size=39949) 2024-11-12T00:53:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T00:53:37,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T00:53:37,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:37,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:37,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:37,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:37,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:37,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372877796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:37,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372877796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:37,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:37,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:37,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:37,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T00:53:37,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:37,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:37,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:37,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:37,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T00:53:37,978 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=591, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/c62b688e8540434f9cbba89d1606db71 2024-11-12T00:53:37,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/479eede7c66a431c8278d0f1f071ca28 is 50, key is test_row_0/B:col10/1731372817147/Put/seqid=0 2024-11-12T00:53:37,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742040_1216 (size=12301) 2024-11-12T00:53:37,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T00:53:37,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:37,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:37,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T00:53:38,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:38,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:38,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:38,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T00:53:38,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T00:53:38,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:38,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:38,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:38,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372878305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:38,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:38,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372878305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:38,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:38,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:38,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:38,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:38,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:38,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/479eede7c66a431c8278d0f1f071ca28 2024-11-12T00:53:38,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/676f6f21791746818b9223d98e010247 is 50, key is test_row_0/C:col10/1731372817147/Put/seqid=0 2024-11-12T00:53:38,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742041_1217 (size=12301) 2024-11-12T00:53:38,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T00:53:38,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:38,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:38,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:38,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T00:53:38,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:38,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:38,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:38,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T00:53:38,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T00:53:38,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:38,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:38,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:38,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:38,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/676f6f21791746818b9223d98e010247 2024-11-12T00:53:38,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/c62b688e8540434f9cbba89d1606db71 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/c62b688e8540434f9cbba89d1606db71 2024-11-12T00:53:38,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/c62b688e8540434f9cbba89d1606db71, entries=200, sequenceid=591, filesize=39.0 K 2024-11-12T00:53:38,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/479eede7c66a431c8278d0f1f071ca28 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/479eede7c66a431c8278d0f1f071ca28 2024-11-12T00:53:38,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/479eede7c66a431c8278d0f1f071ca28, entries=150, sequenceid=591, filesize=12.0 K 2024-11-12T00:53:38,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/676f6f21791746818b9223d98e010247 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/676f6f21791746818b9223d98e010247 2024-11-12T00:53:38,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/676f6f21791746818b9223d98e010247, entries=150, sequenceid=591, filesize=12.0 K 2024-11-12T00:53:38,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~452.20 KB/463056, currentSize=20.13 KB/20610 for 335e77c42adcd1ca9b481da0f68cba79 in 1675ms, sequenceid=591, compaction requested=true 2024-11-12T00:53:38,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:38,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:53:38,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:38,825 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:38,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:53:38,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:38,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:53:38,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:38,825 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:38,826 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143018 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:38,826 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47374 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:38,826 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/A is initiating minor compaction (all files) 2024-11-12T00:53:38,826 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/B is initiating minor compaction (all files) 2024-11-12T00:53:38,826 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/A in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:38,826 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/B in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:38,826 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/a85db80d19414bf8bd5f9045891f2705, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/674ca31cee9e416abf74962cefda3016, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/ff6c95ecb0e84eb29e1488e19430b8b0, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/479eede7c66a431c8278d0f1f071ca28] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=46.3 K 2024-11-12T00:53:38,826 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/bb95059292c449cfb29a008de0350c93, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/1b5a994d719545bfbaf57890a5c2ac55, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/e72fdc94c2894289b51e12d1839e5817, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/c62b688e8540434f9cbba89d1606db71] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=139.7 K 2024-11-12T00:53:38,826 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:38,826 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/bb95059292c449cfb29a008de0350c93, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/1b5a994d719545bfbaf57890a5c2ac55, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/e72fdc94c2894289b51e12d1839e5817, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/c62b688e8540434f9cbba89d1606db71] 2024-11-12T00:53:38,829 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb95059292c449cfb29a008de0350c93, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731372812608 2024-11-12T00:53:38,829 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting a85db80d19414bf8bd5f9045891f2705, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731372812608 2024-11-12T00:53:38,829 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b5a994d719545bfbaf57890a5c2ac55, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731372813312 2024-11-12T00:53:38,829 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 674ca31cee9e416abf74962cefda3016, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731372813312 2024-11-12T00:53:38,830 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting e72fdc94c2894289b51e12d1839e5817, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731372815523 2024-11-12T00:53:38,830 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting ff6c95ecb0e84eb29e1488e19430b8b0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731372815529 2024-11-12T00:53:38,830 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting c62b688e8540434f9cbba89d1606db71, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731372816475 2024-11-12T00:53:38,830 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 479eede7c66a431c8278d0f1f071ca28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731372816475 2024-11-12T00:53:38,842 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:38,845 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112fd16680b47e1474d95a91c5ab08c7ed8_335e77c42adcd1ca9b481da0f68cba79 store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:38,846 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#B#compaction#181 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:38,846 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/840b12d6a1f74228a0fe5cff69dfb650 is 50, key is test_row_0/B:col10/1731372817147/Put/seqid=0 2024-11-12T00:53:38,847 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112fd16680b47e1474d95a91c5ab08c7ed8_335e77c42adcd1ca9b481da0f68cba79, store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:38,848 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fd16680b47e1474d95a91c5ab08c7ed8_335e77c42adcd1ca9b481da0f68cba79 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:38,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742042_1218 (size=13051) 2024-11-12T00:53:38,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742043_1219 (size=4469) 2024-11-12T00:53:38,886 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/840b12d6a1f74228a0fe5cff69dfb650 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/840b12d6a1f74228a0fe5cff69dfb650 2024-11-12T00:53:38,892 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/B of 335e77c42adcd1ca9b481da0f68cba79 into 840b12d6a1f74228a0fe5cff69dfb650(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:38,892 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:38,892 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/B, priority=12, startTime=1731372818825; duration=0sec 2024-11-12T00:53:38,893 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:38,893 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:B 2024-11-12T00:53:38,893 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:53:38,896 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47374 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:53:38,896 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/C is initiating minor compaction (all files) 2024-11-12T00:53:38,896 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/C in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:38,897 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/29ee7296c6a34eb6867284a7ac2017be, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/bf20b6fc53694478bb443e2ec2d0f398, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/f4d5c23cdc9f45fb920ae26bc0407902, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/676f6f21791746818b9223d98e010247] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=46.3 K 2024-11-12T00:53:38,897 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 29ee7296c6a34eb6867284a7ac2017be, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731372812608 2024-11-12T00:53:38,897 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting bf20b6fc53694478bb443e2ec2d0f398, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731372813312 2024-11-12T00:53:38,898 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting f4d5c23cdc9f45fb920ae26bc0407902, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731372815529 2024-11-12T00:53:38,898 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 676f6f21791746818b9223d98e010247, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731372816475 2024-11-12T00:53:38,908 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#C#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:38,908 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/7e7bb43996f440a68785b6d8395773cf is 50, key is test_row_0/C:col10/1731372817147/Put/seqid=0 2024-11-12T00:53:38,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742044_1220 (size=13051) 2024-11-12T00:53:38,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T00:53:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:38,914 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-12T00:53:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:38,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125f42b696b0314d509b8cb0c7b5ada26f_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372817153/Put/seqid=0 2024-11-12T00:53:38,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742045_1221 (size=12454) 2024-11-12T00:53:39,269 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#A#compaction#180 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:39,269 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/738b4963a4404300b67d17c5790c522e is 175, key is test_row_0/A:col10/1731372817147/Put/seqid=0 2024-11-12T00:53:39,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742046_1222 (size=32005) 2024-11-12T00:53:39,279 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/738b4963a4404300b67d17c5790c522e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/738b4963a4404300b67d17c5790c522e 2024-11-12T00:53:39,284 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/A of 335e77c42adcd1ca9b481da0f68cba79 into 738b4963a4404300b67d17c5790c522e(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:39,284 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:39,284 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/A, priority=12, startTime=1731372818825; duration=0sec 2024-11-12T00:53:39,284 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:39,284 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:A 2024-11-12T00:53:39,318 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/7e7bb43996f440a68785b6d8395773cf as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/7e7bb43996f440a68785b6d8395773cf 2024-11-12T00:53:39,323 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/C of 335e77c42adcd1ca9b481da0f68cba79 into 7e7bb43996f440a68785b6d8395773cf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:39,323 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:39,323 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/C, priority=12, startTime=1731372818825; duration=0sec 2024-11-12T00:53:39,323 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:39,323 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:C 2024-11-12T00:53:39,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:39,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. as already flushing 2024-11-12T00:53:39,333 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125f42b696b0314d509b8cb0c7b5ada26f_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125f42b696b0314d509b8cb0c7b5ada26f_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:39,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/9c5bdba23e91493d972b4c97cdcca917, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:39,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/9c5bdba23e91493d972b4c97cdcca917 is 175, key is test_row_0/A:col10/1731372817153/Put/seqid=0 2024-11-12T00:53:39,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742047_1223 (size=31255) 2024-11-12T00:53:39,368 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=598, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/9c5bdba23e91493d972b4c97cdcca917 2024-11-12T00:53:39,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731372819343, max timestamp=1731372819378Num uniques 859; 2024-11-12T00:53:39,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T00:53:39,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372819358, max timestamp=1731372819378Num uniques 859; 2024-11-12T00:53:39,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:53:39,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372819358, max timestamp=1731372819378Num uniques 859; 2024-11-12T00:53:39,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:53:39,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/f16b6af64048422fb0b7cd382dc4adfb is 50, key is test_row_0/B:col10/1731372817153/Put/seqid=0 2024-11-12T00:53:39,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742048_1224 (size=12301) 2024-11-12T00:53:39,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731372819418, max timestamp=1731372819434Num uniques 1018; 2024-11-12T00:53:39,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-12T00:53:39,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731372819418, max timestamp=1731372819434Num uniques 1018; 2024-11-12T00:53:39,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-12T00:53:39,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731372819418, max timestamp=1731372819434Num uniques 1118; 2024-11-12T00:53:39,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-12T00:53:39,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731372819466, max timestamp=1731372819477Num uniques 1127; 2024-11-12T00:53:39,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-12T00:53:39,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731372819466, max timestamp=1731372819477Num uniques 1127; 2024-11-12T00:53:39,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731372819466, max timestamp=1731372819477Num uniques 1127; 2024-11-12T00:53:39,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-12T00:53:39,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-12T00:53:39,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731372819499, max timestamp=1731372819520Num uniques 1036; 2024-11-12T00:53:39,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-12T00:53:39,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731372819499, max timestamp=1731372819520Num uniques 1036; 2024-11-12T00:53:39,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-12T00:53:39,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731372819499, max timestamp=1731372819520Num uniques 1036; 2024-11-12T00:53:39,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-12T00:53:39,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731372819546, max timestamp=1731372819561Num uniques 1095; 2024-11-12T00:53:39,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731372819546, max timestamp=1731372819561Num uniques 1095; 2024-11-12T00:53:39,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-12T00:53:39,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-12T00:53:39,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731372819546, max timestamp=1731372819561Num uniques 1095; 2024-11-12T00:53:39,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-12T00:53:39,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 355 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372879610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:39,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731372819598, max timestamp=1731372819616Num uniques 1204; 2024-11-12T00:53:39,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=11630, suffix heap size=153184,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=254 2024-11-12T00:53:39,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:39,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:39,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:39,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731372819598, max timestamp=1731372819616Num uniques 1204; 2024-11-12T00:53:39,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731372819598, max timestamp=1731372819616Num uniques 1204; 2024-11-12T00:53:39,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=11630, suffix heap size=153184,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=254 2024-11-12T00:53:39,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=11630, suffix heap size=153184,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=254 2024-11-12T00:53:39,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T00:53:39,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731372819697, max timestamp=1731372819719Num uniques 1113; 2024-11-12T00:53:39,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-12T00:53:39,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731372819697, max timestamp=1731372819719Num uniques 1113; 2024-11-12T00:53:39,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-12T00:53:39,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731372819697, max timestamp=1731372819719Num uniques 1113; 2024-11-12T00:53:39,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-12T00:53:39,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731372819747, max timestamp=1731372819779Num uniques 1072; 2024-11-12T00:53:39,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731372819747, max timestamp=1731372819779Num uniques 1072; 2024-11-12T00:53:39,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-12T00:53:39,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-12T00:53:39,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731372819747, max timestamp=1731372819779Num uniques 1072; 2024-11-12T00:53:39,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-12T00:53:39,801 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/f16b6af64048422fb0b7cd382dc4adfb 2024-11-12T00:53:39,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/f7da52b73b614cab8ee30125518ae42c is 50, key is test_row_0/C:col10/1731372817153/Put/seqid=0 2024-11-12T00:53:39,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742049_1225 (size=12301) 2024-11-12T00:53:39,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731372819842, max timestamp=1731372819852Num uniques 1131; 2024-11-12T00:53:39,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-11-12T00:53:39,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731372819842, max timestamp=1731372819852Num uniques 1131; 2024-11-12T00:53:39,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-11-12T00:53:39,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731372819842, max timestamp=1731372819852Num uniques 1131; 2024-11-12T00:53:39,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-11-12T00:53:39,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731372819894, max timestamp=1731372819937Num uniques 1040; 2024-11-12T00:53:39,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-12T00:53:39,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731372819894, max timestamp=1731372819937Num uniques 1040; 2024-11-12T00:53:39,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-12T00:53:39,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731372819894, max timestamp=1731372819937Num uniques 1040; 2024-11-12T00:53:39,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-12T00:53:40,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1731372819986, max timestamp=1731372820006Num uniques 1049; 2024-11-12T00:53:40,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=15984, suffix heap size=137064,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=349 2024-11-12T00:53:40,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1731372819986, max timestamp=1731372820006Num uniques 1049; 2024-11-12T00:53:40,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=15984, suffix heap size=137064,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=349 2024-11-12T00:53:40,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1731372819986, max timestamp=1731372820006Num uniques 1049; 2024-11-12T00:53:40,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=15984, suffix heap size=137064,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=349 2024-11-12T00:53:40,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=308, cellSize=14100, totalHeapSize=32272, min timestamp=1731372820040, max timestamp=1731372820058Num uniques 1208; 2024-11-12T00:53:40,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=55320, new segment data size=14100, suffix heap size=153600,new segment heap size=32272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=308 2024-11-12T00:53:40,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=308, cellSize=14100, totalHeapSize=32272, min timestamp=1731372820040, max timestamp=1731372820058Num uniques 1208; 2024-11-12T00:53:40,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=55320, new segment data size=14100, suffix heap size=153600,new segment heap size=32272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=308 2024-11-12T00:53:40,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=308, cellSize=14100, totalHeapSize=32272, min timestamp=1731372820040, max timestamp=1731372820058Num uniques 1208; 2024-11-12T00:53:40,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=55320, new segment data size=14100, suffix heap size=153600,new segment heap size=32272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=308 2024-11-12T00:53:40,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731372820089, max timestamp=1731372820111Num uniques 1167; 2024-11-12T00:53:40,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53442, new segment data size=9932, suffix heap size=149336,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1167, new segment cells count=217 2024-11-12T00:53:40,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731372820089, max timestamp=1731372820111Num uniques 1167; 2024-11-12T00:53:40,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53442, new segment data size=9932, suffix heap size=149336,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1167, new segment cells count=217 2024-11-12T00:53:40,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731372820089, max timestamp=1731372820111Num uniques 1167; 2024-11-12T00:53:40,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53442, new segment data size=6870, suffix heap size=149336,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1167, new segment cells count=150 2024-11-12T00:53:40,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731372820126, max timestamp=1731372820201Num uniques 1076; 2024-11-12T00:53:40,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-12T00:53:40,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731372820126, max timestamp=1731372820201Num uniques 1076; 2024-11-12T00:53:40,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-12T00:53:40,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731372820126, max timestamp=1731372820201Num uniques 1009; 2024-11-12T00:53:40,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10346, suffix heap size=132904,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=226 2024-11-12T00:53:40,221 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/f7da52b73b614cab8ee30125518ae42c 2024-11-12T00:53:40,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/9c5bdba23e91493d972b4c97cdcca917 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/9c5bdba23e91493d972b4c97cdcca917 2024-11-12T00:53:40,233 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/9c5bdba23e91493d972b4c97cdcca917, entries=150, sequenceid=598, filesize=30.5 K 2024-11-12T00:53:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/f16b6af64048422fb0b7cd382dc4adfb as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f16b6af64048422fb0b7cd382dc4adfb 2024-11-12T00:53:40,242 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f16b6af64048422fb0b7cd382dc4adfb, entries=150, sequenceid=598, filesize=12.0 K 2024-11-12T00:53:40,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/f7da52b73b614cab8ee30125518ae42c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/f7da52b73b614cab8ee30125518ae42c 2024-11-12T00:53:40,247 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/f7da52b73b614cab8ee30125518ae42c, entries=150, sequenceid=598, filesize=12.0 K 2024-11-12T00:53:40,258 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=114.05 KB/116790 for 335e77c42adcd1ca9b481da0f68cba79 in 1343ms, sequenceid=598, compaction requested=false 2024-11-12T00:53:40,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:40,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:40,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-12T00:53:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-12T00:53:40,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=127.47 KB heapSize=358.78 KB 2024-11-12T00:53:40,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:40,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:40,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:40,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:40,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:40,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:53:40,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-12T00:53:40,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7210 sec 2024-11-12T00:53:40,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c3b2d972b67b4db7bd271c1b5505af9a_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372820241/Put/seqid=0 2024-11-12T00:53:40,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.7270 sec 2024-11-12T00:53:40,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742050_1226 (size=17534) 2024-11-12T00:53:40,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:40,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 407 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372880293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 426 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372880293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:40,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:40,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 408 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372880406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 427 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372880406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:40,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:40,410 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-12T00:53:40,410 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:40,410 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:40,410 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:40,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:40,411 INFO [Thread-664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-664. 2024-11-12T00:53:40,411 DEBUG [Thread-664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:40,412 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:40,412 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:40,412 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:40,413 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-11-12T00:53:40,413 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:40,413 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:40,413 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:40,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:40,424 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-12T00:53:40,424 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:40,424 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:40,424 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:40,425 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:40,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:40,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 409 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372880615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 428 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372880616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:40,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:40,679 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:40,683 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c3b2d972b67b4db7bd271c1b5505af9a_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c3b2d972b67b4db7bd271c1b5505af9a_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:40,684 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/0ffc3e8e3cd54b5091a2f30c2d134882, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:40,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/0ffc3e8e3cd54b5091a2f30c2d134882 is 175, key is test_row_0/A:col10/1731372820241/Put/seqid=0 2024-11-12T00:53:40,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742051_1227 (size=48639) 2024-11-12T00:53:40,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:40,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 429 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372880926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 410 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372880925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:41,089 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=858, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/0ffc3e8e3cd54b5091a2f30c2d134882 2024-11-12T00:53:41,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/d0e949f84cc34ca6955374419f4b297b is 50, key is test_row_0/B:col10/1731372820241/Put/seqid=0 2024-11-12T00:53:41,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742052_1228 (size=12301) 2024-11-12T00:53:41,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52768 deadline: 1731372881356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:41,357 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 335e77c42adcd1ca9b481da0f68cba79 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18307 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:41,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:41,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:41,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:41,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52810 deadline: 1731372881386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:41,386 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 335e77c42adcd1ca9b481da0f68cba79 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18333 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:41,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:41,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:41,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:41,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52776 deadline: 1731372881425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:41,427 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 335e77c42adcd1ca9b481da0f68cba79 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18303 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:53:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:41,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 430 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1731372881436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:41,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 411 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52788 deadline: 1731372881436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:53:41,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=858 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/d0e949f84cc34ca6955374419f4b297b 2024-11-12T00:53:41,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/94e66d58050446a296730456cdd2b862 is 50, key is test_row_0/C:col10/1731372820241/Put/seqid=0 2024-11-12T00:53:41,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742053_1229 (size=12301) 2024-11-12T00:53:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T00:53:41,706 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:41,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=858 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/94e66d58050446a296730456cdd2b862 2024-11-12T00:53:41,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/0ffc3e8e3cd54b5091a2f30c2d134882 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/0ffc3e8e3cd54b5091a2f30c2d134882 2024-11-12T00:53:41,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/0ffc3e8e3cd54b5091a2f30c2d134882, entries=250, sequenceid=858, filesize=47.5 K 2024-11-12T00:53:41,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/d0e949f84cc34ca6955374419f4b297b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d0e949f84cc34ca6955374419f4b297b 2024-11-12T00:53:41,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d0e949f84cc34ca6955374419f4b297b, entries=150, sequenceid=858, filesize=12.0 K 2024-11-12T00:53:41,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/94e66d58050446a296730456cdd2b862 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/94e66d58050446a296730456cdd2b862 2024-11-12T00:53:41,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/94e66d58050446a296730456cdd2b862, entries=150, sequenceid=858, filesize=12.0 K 2024-11-12T00:53:41,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~358.73 KB/367344, currentSize=53.67 KB/54960 for 335e77c42adcd1ca9b481da0f68cba79 in 1673ms, sequenceid=858, compaction requested=true 2024-11-12T00:53:41,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:41,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:53:41,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:41,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:53:41,932 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:41,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:41,932 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:41,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 335e77c42adcd1ca9b481da0f68cba79:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:53:41,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:41,933 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:41,933 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:41,933 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/B is initiating minor compaction (all files) 2024-11-12T00:53:41,933 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/A is initiating minor compaction (all files) 2024-11-12T00:53:41,933 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/B in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:41,933 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/A in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:41,933 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/840b12d6a1f74228a0fe5cff69dfb650, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f16b6af64048422fb0b7cd382dc4adfb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d0e949f84cc34ca6955374419f4b297b] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=36.8 K 2024-11-12T00:53:41,933 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/738b4963a4404300b67d17c5790c522e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/9c5bdba23e91493d972b4c97cdcca917, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/0ffc3e8e3cd54b5091a2f30c2d134882] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=109.3 K 2024-11-12T00:53:41,933 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:41,933 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/738b4963a4404300b67d17c5790c522e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/9c5bdba23e91493d972b4c97cdcca917, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/0ffc3e8e3cd54b5091a2f30c2d134882] 2024-11-12T00:53:41,933 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 840b12d6a1f74228a0fe5cff69dfb650, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731372816475 2024-11-12T00:53:41,934 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 738b4963a4404300b67d17c5790c522e, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731372816475 2024-11-12T00:53:41,934 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting f16b6af64048422fb0b7cd382dc4adfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=598, earliestPutTs=1731372817153 2024-11-12T00:53:41,934 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c5bdba23e91493d972b4c97cdcca917, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=598, earliestPutTs=1731372817153 2024-11-12T00:53:41,934 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting d0e949f84cc34ca6955374419f4b297b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=858, earliestPutTs=1731372820226 2024-11-12T00:53:41,934 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ffc3e8e3cd54b5091a2f30c2d134882, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=858, earliestPutTs=1731372820214 2024-11-12T00:53:41,945 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:41,946 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#B#compaction#189 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:41,947 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/64e46679ca634ea6a0d88b235cc586ab is 50, key is test_row_0/B:col10/1731372820241/Put/seqid=0 2024-11-12T00:53:41,948 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111280a0e8aed07b4bff99483ae5013fbf1a_335e77c42adcd1ca9b481da0f68cba79 store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:41,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742054_1230 (size=13153) 2024-11-12T00:53:41,964 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/64e46679ca634ea6a0d88b235cc586ab as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/64e46679ca634ea6a0d88b235cc586ab 2024-11-12T00:53:41,970 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/B of 335e77c42adcd1ca9b481da0f68cba79 into 64e46679ca634ea6a0d88b235cc586ab(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:41,971 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:41,971 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/B, priority=13, startTime=1731372821931; duration=0sec 2024-11-12T00:53:41,971 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:41,971 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:B 2024-11-12T00:53:41,971 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:41,972 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:41,972 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 335e77c42adcd1ca9b481da0f68cba79/C is initiating minor compaction (all files) 2024-11-12T00:53:41,972 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 335e77c42adcd1ca9b481da0f68cba79/C in TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:41,972 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/7e7bb43996f440a68785b6d8395773cf, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/f7da52b73b614cab8ee30125518ae42c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/94e66d58050446a296730456cdd2b862] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp, totalSize=36.8 K 2024-11-12T00:53:41,973 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e7bb43996f440a68785b6d8395773cf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731372816475 2024-11-12T00:53:41,973 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting f7da52b73b614cab8ee30125518ae42c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=598, earliestPutTs=1731372817153 2024-11-12T00:53:41,974 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 94e66d58050446a296730456cdd2b862, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=858, earliestPutTs=1731372820226 2024-11-12T00:53:41,982 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111280a0e8aed07b4bff99483ae5013fbf1a_335e77c42adcd1ca9b481da0f68cba79, store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:41,982 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111280a0e8aed07b4bff99483ae5013fbf1a_335e77c42adcd1ca9b481da0f68cba79 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:41,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742055_1231 (size=4469) 2024-11-12T00:53:41,993 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#A#compaction#190 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:41,994 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/acd5512c341543b6af39f27405f14105 is 175, key is test_row_0/A:col10/1731372820241/Put/seqid=0 2024-11-12T00:53:41,995 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 335e77c42adcd1ca9b481da0f68cba79#C#compaction#191 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:41,995 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/560cbe7f93fa4d149d4b7b705dbecd7f is 50, key is test_row_0/C:col10/1731372820241/Put/seqid=0 2024-11-12T00:53:41,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742056_1232 (size=32107) 2024-11-12T00:53:42,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742057_1233 (size=13153) 2024-11-12T00:53:42,406 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/acd5512c341543b6af39f27405f14105 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/acd5512c341543b6af39f27405f14105 2024-11-12T00:53:42,408 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/560cbe7f93fa4d149d4b7b705dbecd7f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/560cbe7f93fa4d149d4b7b705dbecd7f 2024-11-12T00:53:42,412 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/A of 335e77c42adcd1ca9b481da0f68cba79 into acd5512c341543b6af39f27405f14105(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:42,413 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:42,413 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/A, priority=13, startTime=1731372821931; duration=0sec 2024-11-12T00:53:42,413 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:42,413 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:A 2024-11-12T00:53:42,413 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 335e77c42adcd1ca9b481da0f68cba79/C of 335e77c42adcd1ca9b481da0f68cba79 into 560cbe7f93fa4d149d4b7b705dbecd7f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:42,413 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:42,413 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., storeName=335e77c42adcd1ca9b481da0f68cba79/C, priority=13, startTime=1731372821932; duration=0sec 2024-11-12T00:53:42,413 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:42,413 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 335e77c42adcd1ca9b481da0f68cba79:C 2024-11-12T00:53:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:42,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T00:53:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:42,448 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-11-12T00:53:42,448 INFO [Thread-653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-653. 2024-11-12T00:53:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:42,448 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:42,448 DEBUG [Thread-653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:42,448 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:42,448 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:42,448 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:42,448 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:42,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:42,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:42,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111275bfca0d41234f7cb8def0acd79acac8_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372820272/Put/seqid=0 2024-11-12T00:53:42,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742058_1234 (size=12454) 2024-11-12T00:53:42,865 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:42,869 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111275bfca0d41234f7cb8def0acd79acac8_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111275bfca0d41234f7cb8def0acd79acac8_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:42,870 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/49a4c93b2ab943a1ad68fde45144819b, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:42,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/49a4c93b2ab943a1ad68fde45144819b is 175, key is test_row_0/A:col10/1731372820272/Put/seqid=0 2024-11-12T00:53:42,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742059_1235 (size=31255) 2024-11-12T00:53:43,276 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=874, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/49a4c93b2ab943a1ad68fde45144819b 2024-11-12T00:53:43,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/d7c53002f36a461b89c4291df514b385 is 50, key is test_row_0/B:col10/1731372820272/Put/seqid=0 2024-11-12T00:53:43,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742060_1236 (size=12301) 2024-11-12T00:53:43,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=874 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/d7c53002f36a461b89c4291df514b385 2024-11-12T00:53:43,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/376766665be241b6a87036e4845921a5 is 50, key is test_row_0/C:col10/1731372820272/Put/seqid=0 2024-11-12T00:53:43,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742061_1237 (size=12301) 2024-11-12T00:53:44,054 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T00:53:44,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=874 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/376766665be241b6a87036e4845921a5 2024-11-12T00:53:44,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/49a4c93b2ab943a1ad68fde45144819b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/49a4c93b2ab943a1ad68fde45144819b 2024-11-12T00:53:44,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/49a4c93b2ab943a1ad68fde45144819b, entries=150, sequenceid=874, filesize=30.5 K 2024-11-12T00:53:44,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/d7c53002f36a461b89c4291df514b385 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d7c53002f36a461b89c4291df514b385 2024-11-12T00:53:44,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d7c53002f36a461b89c4291df514b385, entries=150, sequenceid=874, filesize=12.0 K 2024-11-12T00:53:44,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/376766665be241b6a87036e4845921a5 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/376766665be241b6a87036e4845921a5 2024-11-12T00:53:44,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/376766665be241b6a87036e4845921a5, entries=150, sequenceid=874, filesize=12.0 K 2024-11-12T00:53:44,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 335e77c42adcd1ca9b481da0f68cba79 in 1687ms, sequenceid=874, compaction requested=false 2024-11-12T00:53:44,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 335e77c42adcd1ca9b481da0f68cba79: 2024-11-12T00:53:51,367 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-11-12T00:53:51,367 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:51,367 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:51,367 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:51,367 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:51,477 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-12T00:53:51,477 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:51,477 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:51,477 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:51,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:51,527 INFO [Thread-655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-655. 2024-11-12T00:53:51,527 DEBUG [Thread-655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:53:51,527 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:51,527 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:51,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:51,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T00:53:51,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-12T00:53:51,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 357 2024-11-12T00:53:51,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-12T00:53:51,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 372 2024-11-12T00:53:51,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-12T00:53:51,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T00:53:51,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2651 2024-11-12T00:53:51,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2597 2024-11-12T00:53:51,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T00:53:51,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1124 2024-11-12T00:53:51,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3372 rows 2024-11-12T00:53:51,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1124 2024-11-12T00:53:51,528 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3372 rows 2024-11-12T00:53:51,528 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T00:53:51,528 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T00:53:51,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:51,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:51,529 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:53:51,530 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T00:53:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T00:53:51,534 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372831534"}]},"ts":"1731372831534"} 2024-11-12T00:53:51,536 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T00:53:51,536 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T00:53:51,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T00:53:51,538 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=335e77c42adcd1ca9b481da0f68cba79, UNASSIGN}] 2024-11-12T00:53:51,539 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=335e77c42adcd1ca9b481da0f68cba79, UNASSIGN 2024-11-12T00:53:51,539 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=335e77c42adcd1ca9b481da0f68cba79, regionState=CLOSING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:51,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=335e77c42adcd1ca9b481da0f68cba79, UNASSIGN because future has completed 2024-11-12T00:53:51,541 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T00:53:51,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:53:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T00:53:51,694 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(122): Close 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:51,694 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T00:53:51,694 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1722): Closing 335e77c42adcd1ca9b481da0f68cba79, disabling compactions & flushes 2024-11-12T00:53:51,694 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:51,694 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:51,694 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. after waiting 0 ms 2024-11-12T00:53:51,694 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:51,694 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(2902): Flushing 335e77c42adcd1ca9b481da0f68cba79 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-12T00:53:51,695 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=A 2024-11-12T00:53:51,695 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:51,695 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=B 2024-11-12T00:53:51,695 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:51,695 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 335e77c42adcd1ca9b481da0f68cba79, store=C 2024-11-12T00:53:51,695 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:51,705 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122570734d40514805afe48fe6b0087272_335e77c42adcd1ca9b481da0f68cba79 is 50, key is test_row_0/A:col10/1731372831476/Put/seqid=0 2024-11-12T00:53:51,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742062_1238 (size=9914) 2024-11-12T00:53:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T00:53:52,111 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:52,116 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122570734d40514805afe48fe6b0087272_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122570734d40514805afe48fe6b0087272_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:52,117 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/6f618f8216b94a739844e8cd54fc1256, store: [table=TestAcidGuarantees family=A region=335e77c42adcd1ca9b481da0f68cba79] 2024-11-12T00:53:52,117 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/6f618f8216b94a739844e8cd54fc1256 is 175, key is test_row_0/A:col10/1731372831476/Put/seqid=0 2024-11-12T00:53:52,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742063_1239 (size=22561) 2024-11-12T00:53:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T00:53:52,521 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=880, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/6f618f8216b94a739844e8cd54fc1256 2024-11-12T00:53:52,528 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/77070e66436b4874b15c988d5754c3ad is 50, key is test_row_0/B:col10/1731372831476/Put/seqid=0 2024-11-12T00:53:52,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742064_1240 (size=9857) 2024-11-12T00:53:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T00:53:52,940 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=880 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/77070e66436b4874b15c988d5754c3ad 2024-11-12T00:53:52,953 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/7538d7b297ca4637a2ce72807f6294c5 is 50, key is test_row_0/C:col10/1731372831476/Put/seqid=0 2024-11-12T00:53:52,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742065_1241 (size=9857) 2024-11-12T00:53:53,361 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=880 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/7538d7b297ca4637a2ce72807f6294c5 2024-11-12T00:53:53,369 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/A/6f618f8216b94a739844e8cd54fc1256 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/6f618f8216b94a739844e8cd54fc1256 2024-11-12T00:53:53,375 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/6f618f8216b94a739844e8cd54fc1256, entries=100, sequenceid=880, filesize=22.0 K 2024-11-12T00:53:53,376 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/B/77070e66436b4874b15c988d5754c3ad as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/77070e66436b4874b15c988d5754c3ad 2024-11-12T00:53:53,380 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/77070e66436b4874b15c988d5754c3ad, entries=100, sequenceid=880, filesize=9.6 K 2024-11-12T00:53:53,381 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/.tmp/C/7538d7b297ca4637a2ce72807f6294c5 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/7538d7b297ca4637a2ce72807f6294c5 2024-11-12T00:53:53,385 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/7538d7b297ca4637a2ce72807f6294c5, entries=100, sequenceid=880, filesize=9.6 K 2024-11-12T00:53:53,385 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 335e77c42adcd1ca9b481da0f68cba79 in 1691ms, sequenceid=880, compaction requested=true 2024-11-12T00:53:53,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/89e747e5f8bf4e9b8dd91aac552b57d8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/92a9eeda02a7428ca9268f7b388c7aa9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/43ddd530356945ba8b6d2a90b4a696f5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/0eca9cc4c18d41538456eb604ca260c4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/f6b0b36ba70a4270905bcec7602132d8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/6e5be26bb62149ec8bb2c95b94d7996e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/9d10d39fb7fb45609e231947ecafc1f2, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/23b50a68644b457585ccb2506f7afe70, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/f95028444b024b4b8309cf3ad4048e65, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/05eeaaee4a5d4c92b66eb0e00c3fe55e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/feed6cb8bd6d4a8f98d0bf7de5b751bb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/1f740ff96f994933bc1ca3d6e6ee2c30, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/bf6dce4764d64e73bbaecab24ed8c62b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/8165377ec81f4636bd86969897bf5872, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/42c5f6cbdff345ac91191c7c3531e711, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/de4213cd2fef415c8e1fbc74d2741c71, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/3d10242327e648e399cf1f309c477a59, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/908d0d764ace41678c9241c7ba626a4e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/bb95059292c449cfb29a008de0350c93, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/1b5a994d719545bfbaf57890a5c2ac55, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/e72fdc94c2894289b51e12d1839e5817, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/c62b688e8540434f9cbba89d1606db71, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/738b4963a4404300b67d17c5790c522e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/9c5bdba23e91493d972b4c97cdcca917, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/0ffc3e8e3cd54b5091a2f30c2d134882] to archive 2024-11-12T00:53:53,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:53:53,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/89e747e5f8bf4e9b8dd91aac552b57d8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/89e747e5f8bf4e9b8dd91aac552b57d8 2024-11-12T00:53:53,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/92a9eeda02a7428ca9268f7b388c7aa9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/92a9eeda02a7428ca9268f7b388c7aa9 2024-11-12T00:53:53,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/43ddd530356945ba8b6d2a90b4a696f5 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/43ddd530356945ba8b6d2a90b4a696f5 2024-11-12T00:53:53,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/0eca9cc4c18d41538456eb604ca260c4 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/0eca9cc4c18d41538456eb604ca260c4 2024-11-12T00:53:53,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/f6b0b36ba70a4270905bcec7602132d8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/f6b0b36ba70a4270905bcec7602132d8 2024-11-12T00:53:53,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/6e5be26bb62149ec8bb2c95b94d7996e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/6e5be26bb62149ec8bb2c95b94d7996e 2024-11-12T00:53:53,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/9d10d39fb7fb45609e231947ecafc1f2 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/9d10d39fb7fb45609e231947ecafc1f2 2024-11-12T00:53:53,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/23b50a68644b457585ccb2506f7afe70 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/23b50a68644b457585ccb2506f7afe70 2024-11-12T00:53:53,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/f95028444b024b4b8309cf3ad4048e65 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/f95028444b024b4b8309cf3ad4048e65 2024-11-12T00:53:53,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/05eeaaee4a5d4c92b66eb0e00c3fe55e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/05eeaaee4a5d4c92b66eb0e00c3fe55e 2024-11-12T00:53:53,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/feed6cb8bd6d4a8f98d0bf7de5b751bb to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/feed6cb8bd6d4a8f98d0bf7de5b751bb 2024-11-12T00:53:53,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/1f740ff96f994933bc1ca3d6e6ee2c30 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/1f740ff96f994933bc1ca3d6e6ee2c30 2024-11-12T00:53:53,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/bf6dce4764d64e73bbaecab24ed8c62b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/bf6dce4764d64e73bbaecab24ed8c62b 2024-11-12T00:53:53,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/8165377ec81f4636bd86969897bf5872 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/8165377ec81f4636bd86969897bf5872 2024-11-12T00:53:53,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/42c5f6cbdff345ac91191c7c3531e711 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/42c5f6cbdff345ac91191c7c3531e711 2024-11-12T00:53:53,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/de4213cd2fef415c8e1fbc74d2741c71 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/de4213cd2fef415c8e1fbc74d2741c71 2024-11-12T00:53:53,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/3d10242327e648e399cf1f309c477a59 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/3d10242327e648e399cf1f309c477a59 2024-11-12T00:53:53,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/908d0d764ace41678c9241c7ba626a4e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/908d0d764ace41678c9241c7ba626a4e 2024-11-12T00:53:53,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/bb95059292c449cfb29a008de0350c93 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/bb95059292c449cfb29a008de0350c93 2024-11-12T00:53:53,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/1b5a994d719545bfbaf57890a5c2ac55 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/1b5a994d719545bfbaf57890a5c2ac55 2024-11-12T00:53:53,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/e72fdc94c2894289b51e12d1839e5817 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/e72fdc94c2894289b51e12d1839e5817 2024-11-12T00:53:53,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/c62b688e8540434f9cbba89d1606db71 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/c62b688e8540434f9cbba89d1606db71 2024-11-12T00:53:53,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/738b4963a4404300b67d17c5790c522e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/738b4963a4404300b67d17c5790c522e 2024-11-12T00:53:53,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/9c5bdba23e91493d972b4c97cdcca917 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/9c5bdba23e91493d972b4c97cdcca917 2024-11-12T00:53:53,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/0ffc3e8e3cd54b5091a2f30c2d134882 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/0ffc3e8e3cd54b5091a2f30c2d134882 2024-11-12T00:53:53,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/05add621e4444b5e8772717538ce35f9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/451bb14153484514bc2fc6ca3b33b23b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/4789914765cf46daaa5c6ba84e0ecfa0, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/7b15e6cc56014882936ae7563f4ece35, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/42c0b8c9fae64e728acefeb76fb3a3fb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/c311faf738e742c9a297534514e09c26, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/17a8104e162641fdb6817a45398c756f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/18d9f6d7e41f4ad6930c56da166b19d4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/2a0fac81ac484b1d9bca74aa8c36cf0f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/fad942599f524715b73d5b041f0f975a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/e57431fa2fe84ff6a1150ac8fcb5e031, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/062f595b2b3d4bc9868b89b86a4411c8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d674d21ea2874d208b3efd348fc49990, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f3787d1a92ca4c69aa2f28155ab95361, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/90b37b05d0d44535a4398d21418a2e9c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/879f69cd9cfa4533b08507160c8aa53e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f0accbeb10984392b5fcc9111f14d7d4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/a85db80d19414bf8bd5f9045891f2705, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/37f112df6fe84f18b9a838db277afbd0, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/674ca31cee9e416abf74962cefda3016, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/ff6c95ecb0e84eb29e1488e19430b8b0, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/840b12d6a1f74228a0fe5cff69dfb650, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/479eede7c66a431c8278d0f1f071ca28, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f16b6af64048422fb0b7cd382dc4adfb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d0e949f84cc34ca6955374419f4b297b] to archive 2024-11-12T00:53:53,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:53:53,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/05add621e4444b5e8772717538ce35f9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/05add621e4444b5e8772717538ce35f9 2024-11-12T00:53:53,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/451bb14153484514bc2fc6ca3b33b23b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/451bb14153484514bc2fc6ca3b33b23b 2024-11-12T00:53:53,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/4789914765cf46daaa5c6ba84e0ecfa0 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/4789914765cf46daaa5c6ba84e0ecfa0 2024-11-12T00:53:53,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/7b15e6cc56014882936ae7563f4ece35 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/7b15e6cc56014882936ae7563f4ece35 2024-11-12T00:53:53,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/42c0b8c9fae64e728acefeb76fb3a3fb to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/42c0b8c9fae64e728acefeb76fb3a3fb 2024-11-12T00:53:53,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/c311faf738e742c9a297534514e09c26 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/c311faf738e742c9a297534514e09c26 2024-11-12T00:53:53,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/17a8104e162641fdb6817a45398c756f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/17a8104e162641fdb6817a45398c756f 2024-11-12T00:53:53,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/18d9f6d7e41f4ad6930c56da166b19d4 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/18d9f6d7e41f4ad6930c56da166b19d4 2024-11-12T00:53:53,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/2a0fac81ac484b1d9bca74aa8c36cf0f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/2a0fac81ac484b1d9bca74aa8c36cf0f 2024-11-12T00:53:53,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/fad942599f524715b73d5b041f0f975a to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/fad942599f524715b73d5b041f0f975a 2024-11-12T00:53:53,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/e57431fa2fe84ff6a1150ac8fcb5e031 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/e57431fa2fe84ff6a1150ac8fcb5e031 2024-11-12T00:53:53,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/062f595b2b3d4bc9868b89b86a4411c8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/062f595b2b3d4bc9868b89b86a4411c8 2024-11-12T00:53:53,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d674d21ea2874d208b3efd348fc49990 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d674d21ea2874d208b3efd348fc49990 2024-11-12T00:53:53,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f3787d1a92ca4c69aa2f28155ab95361 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f3787d1a92ca4c69aa2f28155ab95361 2024-11-12T00:53:53,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/90b37b05d0d44535a4398d21418a2e9c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/90b37b05d0d44535a4398d21418a2e9c 2024-11-12T00:53:53,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/879f69cd9cfa4533b08507160c8aa53e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/879f69cd9cfa4533b08507160c8aa53e 2024-11-12T00:53:53,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f0accbeb10984392b5fcc9111f14d7d4 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f0accbeb10984392b5fcc9111f14d7d4 2024-11-12T00:53:53,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/a85db80d19414bf8bd5f9045891f2705 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/a85db80d19414bf8bd5f9045891f2705 2024-11-12T00:53:53,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/37f112df6fe84f18b9a838db277afbd0 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/37f112df6fe84f18b9a838db277afbd0 2024-11-12T00:53:53,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/674ca31cee9e416abf74962cefda3016 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/674ca31cee9e416abf74962cefda3016 2024-11-12T00:53:53,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/ff6c95ecb0e84eb29e1488e19430b8b0 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/ff6c95ecb0e84eb29e1488e19430b8b0 2024-11-12T00:53:53,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/840b12d6a1f74228a0fe5cff69dfb650 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/840b12d6a1f74228a0fe5cff69dfb650 2024-11-12T00:53:53,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/479eede7c66a431c8278d0f1f071ca28 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/479eede7c66a431c8278d0f1f071ca28 2024-11-12T00:53:53,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f16b6af64048422fb0b7cd382dc4adfb to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/f16b6af64048422fb0b7cd382dc4adfb 2024-11-12T00:53:53,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d0e949f84cc34ca6955374419f4b297b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d0e949f84cc34ca6955374419f4b297b 2024-11-12T00:53:53,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/0c5fdc1ff1ee4021bc2d71a4178170de, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/22f361dc6c484a29affe5b477ee90ebe, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/3312e7eb3c9041669e15bc8b2a60b6dc, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/b02914e06e354374b37f2b080b28e63e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/8388ca7b23e14c909c4ccfaa7631bc52, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/355be2668fcc4aaea7b57aa2d591cca3, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/23fe35e20b214a5da3e0e2119840bb29, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/0a103485fd5e40c3a014c9963c4840c7, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/4940e463eb2344fe8e0f0dce2c689713, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/fb5f2f00b97142dba773807ec32107eb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/01972d5e9d8d4c60bf4d15e2c5dcb14e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/8d7c183ba41f4725b9c65283bd8f2d80, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/21727f45d69a413886428bd7e56a474e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/9716ab32023c474583ec2f1f5a3bc740, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/77d6deb05ef24da28123501dbf331faf, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/ffc439bc197d433b920bc2158d595683, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/09f28b34d28047808d1be4a660a31d76, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/29ee7296c6a34eb6867284a7ac2017be, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/af705477c8f64d14972e40203a1b016d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/bf20b6fc53694478bb443e2ec2d0f398, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/f4d5c23cdc9f45fb920ae26bc0407902, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/7e7bb43996f440a68785b6d8395773cf, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/676f6f21791746818b9223d98e010247, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/f7da52b73b614cab8ee30125518ae42c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/94e66d58050446a296730456cdd2b862] to archive 2024-11-12T00:53:53,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:53:53,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/0c5fdc1ff1ee4021bc2d71a4178170de to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/0c5fdc1ff1ee4021bc2d71a4178170de 2024-11-12T00:53:53,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/22f361dc6c484a29affe5b477ee90ebe to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/22f361dc6c484a29affe5b477ee90ebe 2024-11-12T00:53:53,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/3312e7eb3c9041669e15bc8b2a60b6dc to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/3312e7eb3c9041669e15bc8b2a60b6dc 2024-11-12T00:53:53,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/b02914e06e354374b37f2b080b28e63e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/b02914e06e354374b37f2b080b28e63e 2024-11-12T00:53:53,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/8388ca7b23e14c909c4ccfaa7631bc52 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/8388ca7b23e14c909c4ccfaa7631bc52 2024-11-12T00:53:53,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/355be2668fcc4aaea7b57aa2d591cca3 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/355be2668fcc4aaea7b57aa2d591cca3 2024-11-12T00:53:53,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/23fe35e20b214a5da3e0e2119840bb29 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/23fe35e20b214a5da3e0e2119840bb29 2024-11-12T00:53:53,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/0a103485fd5e40c3a014c9963c4840c7 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/0a103485fd5e40c3a014c9963c4840c7 2024-11-12T00:53:53,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/4940e463eb2344fe8e0f0dce2c689713 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/4940e463eb2344fe8e0f0dce2c689713 2024-11-12T00:53:53,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/fb5f2f00b97142dba773807ec32107eb to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/fb5f2f00b97142dba773807ec32107eb 2024-11-12T00:53:53,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/01972d5e9d8d4c60bf4d15e2c5dcb14e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/01972d5e9d8d4c60bf4d15e2c5dcb14e 2024-11-12T00:53:53,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/8d7c183ba41f4725b9c65283bd8f2d80 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/8d7c183ba41f4725b9c65283bd8f2d80 2024-11-12T00:53:53,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/21727f45d69a413886428bd7e56a474e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/21727f45d69a413886428bd7e56a474e 2024-11-12T00:53:53,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/9716ab32023c474583ec2f1f5a3bc740 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/9716ab32023c474583ec2f1f5a3bc740 2024-11-12T00:53:53,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/77d6deb05ef24da28123501dbf331faf to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/77d6deb05ef24da28123501dbf331faf 2024-11-12T00:53:53,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/ffc439bc197d433b920bc2158d595683 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/ffc439bc197d433b920bc2158d595683 2024-11-12T00:53:53,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/09f28b34d28047808d1be4a660a31d76 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/09f28b34d28047808d1be4a660a31d76 2024-11-12T00:53:53,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/29ee7296c6a34eb6867284a7ac2017be to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/29ee7296c6a34eb6867284a7ac2017be 2024-11-12T00:53:53,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/af705477c8f64d14972e40203a1b016d to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/af705477c8f64d14972e40203a1b016d 2024-11-12T00:53:53,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/bf20b6fc53694478bb443e2ec2d0f398 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/bf20b6fc53694478bb443e2ec2d0f398 2024-11-12T00:53:53,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/f4d5c23cdc9f45fb920ae26bc0407902 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/f4d5c23cdc9f45fb920ae26bc0407902 2024-11-12T00:53:53,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/7e7bb43996f440a68785b6d8395773cf to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/7e7bb43996f440a68785b6d8395773cf 2024-11-12T00:53:53,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/676f6f21791746818b9223d98e010247 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/676f6f21791746818b9223d98e010247 2024-11-12T00:53:53,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/f7da52b73b614cab8ee30125518ae42c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/f7da52b73b614cab8ee30125518ae42c 2024-11-12T00:53:53,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/94e66d58050446a296730456cdd2b862 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/94e66d58050446a296730456cdd2b862 2024-11-12T00:53:53,485 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/recovered.edits/883.seqid, newMaxSeqId=883, maxSeqId=4 2024-11-12T00:53:53,486 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. 2024-11-12T00:53:53,486 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1676): Region close journal for 335e77c42adcd1ca9b481da0f68cba79: Waiting for close lock at 1731372831694Running coprocessor pre-close hooks at 1731372831694Disabling compacts and flushes for region at 1731372831694Disabling writes for close at 1731372831694Obtaining lock to block concurrent updates at 1731372831694Preparing flush snapshotting stores in 335e77c42adcd1ca9b481da0f68cba79 at 1731372831694Finished memstore snapshotting TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731372831695 (+1 ms)Flushing stores of TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79. at 1731372831696 (+1 ms)Flushing 335e77c42adcd1ca9b481da0f68cba79/A: creating writer at 1731372831696Flushing mob file 335e77c42adcd1ca9b481da0f68cba79/A: appending metadata at 1731372831705 (+9 ms)Flushing mob file 335e77c42adcd1ca9b481da0f68cba79/A: closing flushed file at 1731372831705Flushing 335e77c42adcd1ca9b481da0f68cba79/A: appending metadata at 1731372832117 (+412 ms)Flushing 335e77c42adcd1ca9b481da0f68cba79/A: closing flushed file at 1731372832117Flushing 335e77c42adcd1ca9b481da0f68cba79/B: creating writer at 1731372832525 (+408 ms)Flushing 335e77c42adcd1ca9b481da0f68cba79/B: appending metadata at 1731372832528 (+3 ms)Flushing 335e77c42adcd1ca9b481da0f68cba79/B: closing flushed file at 1731372832528Flushing 335e77c42adcd1ca9b481da0f68cba79/C: creating writer at 1731372832947 (+419 ms)Flushing 335e77c42adcd1ca9b481da0f68cba79/C: appending metadata at 1731372832952 (+5 ms)Flushing 335e77c42adcd1ca9b481da0f68cba79/C: closing flushed file at 1731372832952Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40e66edb: reopening flushed file at 1731372833368 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f1879dc: reopening flushed file at 1731372833375 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62439b24: reopening flushed file at 1731372833380 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 335e77c42adcd1ca9b481da0f68cba79 in 1691ms, sequenceid=880, compaction requested=true at 1731372833385 (+5 ms)Writing region close event to WAL at 1731372833482 (+97 ms)Running coprocessor post-close hooks at 1731372833485 (+3 ms)Closed at 1731372833486 (+1 ms) 2024-11-12T00:53:53,488 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=335e77c42adcd1ca9b481da0f68cba79, regionState=CLOSED 2024-11-12T00:53:53,489 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(157): Closed 335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:53:53,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-12T00:53:53,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseRegionProcedure 335e77c42adcd1ca9b481da0f68cba79, server=6928b69dca1b,33953,1731372765559 in 1.9490 sec 2024-11-12T00:53:53,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-11-12T00:53:53,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=335e77c42adcd1ca9b481da0f68cba79, UNASSIGN in 1.9540 sec 2024-11-12T00:53:53,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-12T00:53:53,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9570 sec 2024-11-12T00:53:53,496 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372833496"}]},"ts":"1731372833496"} 2024-11-12T00:53:53,497 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T00:53:53,497 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T00:53:53,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9680 sec 2024-11-12T00:53:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T00:53:53,677 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:53,677 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T00:53:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:53,680 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T00:53:53,681 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=52, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:53,684 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,686 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/recovered.edits] 2024-11-12T00:53:53,689 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/49a4c93b2ab943a1ad68fde45144819b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/49a4c93b2ab943a1ad68fde45144819b 2024-11-12T00:53:53,691 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/6f618f8216b94a739844e8cd54fc1256 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/6f618f8216b94a739844e8cd54fc1256 2024-11-12T00:53:53,692 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/acd5512c341543b6af39f27405f14105 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/A/acd5512c341543b6af39f27405f14105 2024-11-12T00:53:53,695 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/64e46679ca634ea6a0d88b235cc586ab to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/64e46679ca634ea6a0d88b235cc586ab 2024-11-12T00:53:53,696 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/77070e66436b4874b15c988d5754c3ad to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/77070e66436b4874b15c988d5754c3ad 2024-11-12T00:53:53,697 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d7c53002f36a461b89c4291df514b385 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/B/d7c53002f36a461b89c4291df514b385 2024-11-12T00:53:53,699 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/376766665be241b6a87036e4845921a5 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/376766665be241b6a87036e4845921a5 2024-11-12T00:53:53,700 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/560cbe7f93fa4d149d4b7b705dbecd7f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/560cbe7f93fa4d149d4b7b705dbecd7f 2024-11-12T00:53:53,701 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/7538d7b297ca4637a2ce72807f6294c5 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/C/7538d7b297ca4637a2ce72807f6294c5 2024-11-12T00:53:53,703 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/recovered.edits/883.seqid to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79/recovered.edits/883.seqid 2024-11-12T00:53:53,704 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,704 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T00:53:53,704 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T00:53:53,705 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T00:53:53,709 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112123867a816ee433183a87c1f199ed606_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112123867a816ee433183a87c1f199ed606_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,710 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111214a8bee817e04f90a000be3107ad69b8_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111214a8bee817e04f90a000be3107ad69b8_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,711 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122570734d40514805afe48fe6b0087272_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122570734d40514805afe48fe6b0087272_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,712 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122b12b8e196c44ba0939db48a74715e32_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122b12b8e196c44ba0939db48a74715e32_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,713 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111244aa872e1ee749e6907f5b4fffd332b2_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111244aa872e1ee749e6907f5b4fffd332b2_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,714 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112562c1a091e4f45cf87f3ebd7124ae481_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112562c1a091e4f45cf87f3ebd7124ae481_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,715 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125f42b696b0314d509b8cb0c7b5ada26f_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125f42b696b0314d509b8cb0c7b5ada26f_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,716 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111275bfca0d41234f7cb8def0acd79acac8_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111275bfca0d41234f7cb8def0acd79acac8_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,717 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112877b0d245f1049a994a12c5469093e00_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112877b0d245f1049a994a12c5469093e00_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,718 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112889dc6e7e07e41d6b176b25359cd68e4_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112889dc6e7e07e41d6b176b25359cd68e4_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,719 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111293c8b6d08b40434ca2d98aca284d4b5a_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111293c8b6d08b40434ca2d98aca284d4b5a_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,720 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111294187c33a9eb4a86bee5b76115ecc283_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111294187c33a9eb4a86bee5b76115ecc283_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,721 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a150fe5bf50d436984706a0f221036ec_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a150fe5bf50d436984706a0f221036ec_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,722 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a4c7b149745a480f9320f0b1256f6115_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a4c7b149745a480f9320f0b1256f6115_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,723 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a4fdac4e6de74d57b73e1e8f4ed84ce1_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a4fdac4e6de74d57b73e1e8f4ed84ce1_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,724 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b281caff91684412b5c0443637809861_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b281caff91684412b5c0443637809861_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,725 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c3b2d972b67b4db7bd271c1b5505af9a_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c3b2d972b67b4db7bd271c1b5505af9a_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,726 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c91d6ab3e6174f53833bbe43f140c91f_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c91d6ab3e6174f53833bbe43f140c91f_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,727 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f1f3540092d14779b5f92dd58b357635_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f1f3540092d14779b5f92dd58b357635_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,728 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112fb66a25415a54e88ada1d639ff96fc9c_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112fb66a25415a54e88ada1d639ff96fc9c_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,729 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112fdf9560943e941258331a66f203d09dc_335e77c42adcd1ca9b481da0f68cba79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112fdf9560943e941258331a66f203d09dc_335e77c42adcd1ca9b481da0f68cba79 2024-11-12T00:53:53,730 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T00:53:53,732 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=52, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:53,735 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T00:53:53,737 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T00:53:53,738 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=52, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:53,738 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T00:53:53,738 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731372833738"}]},"ts":"9223372036854775807"} 2024-11-12T00:53:53,740 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T00:53:53,740 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 335e77c42adcd1ca9b481da0f68cba79, NAME => 'TestAcidGuarantees,,1731372797183.335e77c42adcd1ca9b481da0f68cba79.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T00:53:53,740 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T00:53:53,740 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731372833740"}]},"ts":"9223372036854775807"} 2024-11-12T00:53:53,742 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T00:53:53,743 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=52, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:53,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2024-11-12T00:53:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T00:53:53,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T00:53:53,786 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:53,797 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 249), OpenFileDescriptor=449 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=430 (was 532), ProcessCount=11 (was 11), AvailableMemoryMB=4817 (was 5668) 2024-11-12T00:53:53,807 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=430, ProcessCount=11, AvailableMemoryMB=4817 2024-11-12T00:53:53,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T00:53:53,809 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T00:53:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T00:53:53,811 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T00:53:53,811 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:53,812 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 53 2024-11-12T00:53:53,812 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T00:53:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-12T00:53:53,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742066_1242 (size=1061) 2024-11-12T00:53:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-12T00:53:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-12T00:53:54,221 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6 2024-11-12T00:53:54,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742067_1243 (size=53) 2024-11-12T00:53:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-12T00:53:54,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:53:54,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing eacf29d404b7a11371b9ac1286c14c2b, disabling compactions & flushes 2024-11-12T00:53:54,630 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:54,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:54,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. after waiting 0 ms 2024-11-12T00:53:54,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:54,630 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:54,631 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for eacf29d404b7a11371b9ac1286c14c2b: Waiting for close lock at 1731372834630Disabling compacts and flushes for region at 1731372834630Disabling writes for close at 1731372834630Writing region close event to WAL at 1731372834630Closed at 1731372834630 2024-11-12T00:53:54,632 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T00:53:54,633 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731372834633"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731372834633"}]},"ts":"1731372834633"} 2024-11-12T00:53:54,636 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T00:53:54,638 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T00:53:54,638 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372834638"}]},"ts":"1731372834638"} 2024-11-12T00:53:54,641 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T00:53:54,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eacf29d404b7a11371b9ac1286c14c2b, ASSIGN}] 2024-11-12T00:53:54,642 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eacf29d404b7a11371b9ac1286c14c2b, ASSIGN 2024-11-12T00:53:54,643 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eacf29d404b7a11371b9ac1286c14c2b, ASSIGN; state=OFFLINE, location=6928b69dca1b,33953,1731372765559; forceNewPlan=false, retain=false 2024-11-12T00:53:54,794 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=eacf29d404b7a11371b9ac1286c14c2b, regionState=OPENING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:54,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eacf29d404b7a11371b9ac1286c14c2b, ASSIGN because future has completed 2024-11-12T00:53:54,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:53:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-12T00:53:54,959 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:54,959 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} 2024-11-12T00:53:54,960 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:54,960 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:53:54,960 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:54,960 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:54,963 INFO [StoreOpener-eacf29d404b7a11371b9ac1286c14c2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:54,966 INFO [StoreOpener-eacf29d404b7a11371b9ac1286c14c2b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:53:54,966 INFO [StoreOpener-eacf29d404b7a11371b9ac1286c14c2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eacf29d404b7a11371b9ac1286c14c2b columnFamilyName A 2024-11-12T00:53:54,966 DEBUG [StoreOpener-eacf29d404b7a11371b9ac1286c14c2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:54,967 INFO [StoreOpener-eacf29d404b7a11371b9ac1286c14c2b-1 {}] regionserver.HStore(327): Store=eacf29d404b7a11371b9ac1286c14c2b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:53:54,967 INFO [StoreOpener-eacf29d404b7a11371b9ac1286c14c2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:54,968 INFO [StoreOpener-eacf29d404b7a11371b9ac1286c14c2b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:53:54,969 INFO [StoreOpener-eacf29d404b7a11371b9ac1286c14c2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eacf29d404b7a11371b9ac1286c14c2b columnFamilyName B 2024-11-12T00:53:54,969 DEBUG [StoreOpener-eacf29d404b7a11371b9ac1286c14c2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:54,969 INFO [StoreOpener-eacf29d404b7a11371b9ac1286c14c2b-1 {}] regionserver.HStore(327): Store=eacf29d404b7a11371b9ac1286c14c2b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:53:54,969 INFO [StoreOpener-eacf29d404b7a11371b9ac1286c14c2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:54,971 INFO [StoreOpener-eacf29d404b7a11371b9ac1286c14c2b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:53:54,971 INFO [StoreOpener-eacf29d404b7a11371b9ac1286c14c2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eacf29d404b7a11371b9ac1286c14c2b columnFamilyName C 2024-11-12T00:53:54,971 DEBUG [StoreOpener-eacf29d404b7a11371b9ac1286c14c2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:53:54,972 INFO [StoreOpener-eacf29d404b7a11371b9ac1286c14c2b-1 {}] regionserver.HStore(327): Store=eacf29d404b7a11371b9ac1286c14c2b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:53:54,972 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:54,972 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:54,973 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:54,974 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:54,975 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:54,975 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:54,976 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T00:53:54,977 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:54,979 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T00:53:54,979 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened eacf29d404b7a11371b9ac1286c14c2b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62233998, jitterRate=-0.07264116406440735}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T00:53:54,980 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:54,980 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for eacf29d404b7a11371b9ac1286c14c2b: Running coprocessor pre-open hook at 1731372834961Writing region info on filesystem at 1731372834961Initializing all the Stores at 1731372834962 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372834963 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372834963Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372834963Cleaning up temporary data from old regions at 1731372834975 (+12 ms)Running coprocessor post-open hooks at 1731372834980 (+5 ms)Region opened successfully at 1731372834980 2024-11-12T00:53:54,981 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., pid=55, masterSystemTime=1731372834953 2024-11-12T00:53:54,982 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:54,982 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:54,983 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=eacf29d404b7a11371b9ac1286c14c2b, regionState=OPEN, openSeqNum=2, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:54,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:53:54,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-12T00:53:54,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; OpenRegionProcedure eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 in 186 msec 2024-11-12T00:53:54,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-12T00:53:54,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eacf29d404b7a11371b9ac1286c14c2b, ASSIGN in 345 msec 2024-11-12T00:53:54,988 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T00:53:54,988 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372834988"}]},"ts":"1731372834988"} 2024-11-12T00:53:54,989 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T00:53:54,990 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T00:53:54,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-11-12T00:53:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-12T00:53:55,956 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:55,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a1f6c3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:55,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:55,961 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6606d8e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:55,964 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f031bc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,966 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:55,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:55,970 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T00:53:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 6928b69dca1b,45925,1731372764943 2024-11-12T00:53:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7fabc20d 2024-11-12T00:53:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T00:53:55,972 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T00:53:55,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555e4e20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,974 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:55,974 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:55,975 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@777452f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:55,977 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33b33a66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555b0fec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,979 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:55,979 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:55,980 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b2c52b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:55,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:55,982 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:55,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55077326, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:55,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:55,984 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c6de51e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:55,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:55,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b921486, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:55,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:55,988 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c2a3ecc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:55,990 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2501e188, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:55,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:55,993 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60d0b926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:55,995 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:55,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45a4e644, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:55,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:55,997 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:55,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:55,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:55,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4626d2e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:55,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:55,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:55,998 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c60b35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:55,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:55,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:56,000 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1737a8b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:56,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:56,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:56,001 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a67f89d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:56,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:56,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:56,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:56,003 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:56,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:56,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:56,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31543fff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:56,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:56,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:56,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:56,004 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41adfd91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:56,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:56,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:56,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:56,006 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e29668, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:56,007 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:56,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68912f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:56,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:56,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:53:56,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:53:56,008 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:53:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:53:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:53:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b1b90e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:53:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:53:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:53:56,010 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:53:56,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2982107b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:53:56,011 DEBUG [Thread-1146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:56,011 DEBUG [Thread-1148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:56,011 DEBUG [Thread-1142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:56,011 DEBUG [Thread-1133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:56,011 DEBUG [Thread-1135 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:56,011 DEBUG [Thread-1144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:56,011 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:56,012 DEBUG [Thread-1148 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:56,012 DEBUG [Thread-1135 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:56,012 DEBUG [Thread-1148 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:56,013 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:56,013 DEBUG [Thread-1131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:56,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:56,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:56,014 DEBUG [Thread-1150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:56,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:56,014 DEBUG [Thread-1139 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:53:56,014 DEBUG [Thread-1135 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:56,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-12T00:53:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:56,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:56,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:56,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:53:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:53:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:53:56,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:56,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T00:53:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:53:56,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:53:56,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:53:56,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:53:56,019 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:56,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:53:56,020 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:56,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:53:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:53:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:53:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:53:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:53:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:56,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T00:53:56,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:53:56,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:56,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:53:56,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:56,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:53:56,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:56,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/dc7b0df68777418d8a6efc80e8ea95df is 50, key is test_row_0/A:col10/1731372836031/Put/seqid=0 2024-11-12T00:53:56,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372896060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372896060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742068_1244 (size=12001) 2024-11-12T00:53:56,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372896065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372896066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372896067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T00:53:56,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-12T00:53:56,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:56,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:53:56,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:56,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:56,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:56,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372896175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372896176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372896176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372896176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372896176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-12T00:53:56,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:56,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:53:56,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:56,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:56,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T00:53:56,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372896385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372896386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372896386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372896386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372896386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/dc7b0df68777418d8a6efc80e8ea95df 2024-11-12T00:53:56,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-12T00:53:56,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:56,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:53:56,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:56,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:56,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:53:56,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/14054e439a0f4e379217252df1aea48c is 50, key is test_row_0/B:col10/1731372836031/Put/seqid=0 2024-11-12T00:53:56,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742069_1245 (size=12001) 2024-11-12T00:53:56,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/14054e439a0f4e379217252df1aea48c 2024-11-12T00:53:56,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/437487d892614748b9e216b082e90631 is 50, key is test_row_0/C:col10/1731372836031/Put/seqid=0 2024-11-12T00:53:56,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742070_1246 (size=12001) 2024-11-12T00:53:56,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/437487d892614748b9e216b082e90631 2024-11-12T00:53:56,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/dc7b0df68777418d8a6efc80e8ea95df as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/dc7b0df68777418d8a6efc80e8ea95df 2024-11-12T00:53:56,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/dc7b0df68777418d8a6efc80e8ea95df, entries=150, sequenceid=15, filesize=11.7 K 2024-11-12T00:53:56,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/14054e439a0f4e379217252df1aea48c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/14054e439a0f4e379217252df1aea48c 2024-11-12T00:53:56,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/14054e439a0f4e379217252df1aea48c, entries=150, sequenceid=15, filesize=11.7 K 2024-11-12T00:53:56,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/437487d892614748b9e216b082e90631 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/437487d892614748b9e216b082e90631 2024-11-12T00:53:56,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/437487d892614748b9e216b082e90631, entries=150, sequenceid=15, filesize=11.7 K 2024-11-12T00:53:56,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for eacf29d404b7a11371b9ac1286c14c2b in 535ms, sequenceid=15, compaction requested=false 2024-11-12T00:53:56,567 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T00:53:56,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:53:56,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-12T00:53:56,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:56,638 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T00:53:56,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:53:56,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:56,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:53:56,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:56,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:53:56,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T00:53:56,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/a91eccadcbf44278bdbea0511ee3ae40 is 50, key is test_row_0/A:col10/1731372836063/Put/seqid=0 2024-11-12T00:53:56,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742071_1247 (size=12001) 2024-11-12T00:53:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:56,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:53:56,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372896704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372896704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372896705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372896705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372896706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372896816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372896816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372896816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372896816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:56,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372896816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372897025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372897025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372897026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372897026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372897026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,058 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/a91eccadcbf44278bdbea0511ee3ae40 2024-11-12T00:53:57,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/7867a159f5b541808f69061f296f9680 is 50, key is test_row_0/B:col10/1731372836063/Put/seqid=0 2024-11-12T00:53:57,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742072_1248 (size=12001) 2024-11-12T00:53:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T00:53:57,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372897335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372897335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372897335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372897335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372897335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,479 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/7867a159f5b541808f69061f296f9680 2024-11-12T00:53:57,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/1df66fa9b90b4e61966f91c30a328e48 is 50, key is test_row_0/C:col10/1731372836063/Put/seqid=0 2024-11-12T00:53:57,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742073_1249 (size=12001) 2024-11-12T00:53:57,503 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/1df66fa9b90b4e61966f91c30a328e48 2024-11-12T00:53:57,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/a91eccadcbf44278bdbea0511ee3ae40 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/a91eccadcbf44278bdbea0511ee3ae40 2024-11-12T00:53:57,520 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/a91eccadcbf44278bdbea0511ee3ae40, entries=150, sequenceid=33, filesize=11.7 K 2024-11-12T00:53:57,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/7867a159f5b541808f69061f296f9680 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/7867a159f5b541808f69061f296f9680 2024-11-12T00:53:57,527 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/7867a159f5b541808f69061f296f9680, entries=150, sequenceid=33, filesize=11.7 K 2024-11-12T00:53:57,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/1df66fa9b90b4e61966f91c30a328e48 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/1df66fa9b90b4e61966f91c30a328e48 2024-11-12T00:53:57,534 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/1df66fa9b90b4e61966f91c30a328e48, entries=150, sequenceid=33, filesize=11.7 K 2024-11-12T00:53:57,535 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for eacf29d404b7a11371b9ac1286c14c2b in 897ms, sequenceid=33, compaction requested=false 2024-11-12T00:53:57,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:53:57,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:57,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-12T00:53:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-12T00:53:57,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-12T00:53:57,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5180 sec 2024-11-12T00:53:57,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.5280 sec 2024-11-12T00:53:57,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:57,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T00:53:57,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:53:57,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:57,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:53:57,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:57,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:53:57,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:57,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/63c4a59f03374dddb1877bebf5624918 is 50, key is test_row_0/A:col10/1731372836697/Put/seqid=0 2024-11-12T00:53:57,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742074_1250 (size=14341) 2024-11-12T00:53:57,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/63c4a59f03374dddb1877bebf5624918 2024-11-12T00:53:57,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372897862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372897864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372897866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372897868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372897869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/e17e292b467a49a3b649076ad28bbe48 is 50, key is test_row_0/B:col10/1731372836697/Put/seqid=0 2024-11-12T00:53:57,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742075_1251 (size=12001) 2024-11-12T00:53:57,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372897975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372897975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372897975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372897985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372897986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T00:53:58,165 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:58,166 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-12T00:53:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T00:53:58,167 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:58,168 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:58,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:58,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372898186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372898186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372898186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372898196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372898196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T00:53:58,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/e17e292b467a49a3b649076ad28bbe48 2024-11-12T00:53:58,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/9a478915c1184145831919ab233b83dc is 50, key is test_row_0/C:col10/1731372836697/Put/seqid=0 2024-11-12T00:53:58,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742076_1252 (size=12001) 2024-11-12T00:53:58,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/9a478915c1184145831919ab233b83dc 2024-11-12T00:53:58,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/63c4a59f03374dddb1877bebf5624918 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/63c4a59f03374dddb1877bebf5624918 2024-11-12T00:53:58,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/63c4a59f03374dddb1877bebf5624918, entries=200, sequenceid=48, filesize=14.0 K 2024-11-12T00:53:58,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/e17e292b467a49a3b649076ad28bbe48 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/e17e292b467a49a3b649076ad28bbe48 2024-11-12T00:53:58,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/e17e292b467a49a3b649076ad28bbe48, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T00:53:58,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/9a478915c1184145831919ab233b83dc as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9a478915c1184145831919ab233b83dc 2024-11-12T00:53:58,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9a478915c1184145831919ab233b83dc, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T00:53:58,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for eacf29d404b7a11371b9ac1286c14c2b in 467ms, sequenceid=48, compaction requested=true 2024-11-12T00:53:58,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:53:58,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:53:58,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:58,314 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:58,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:53:58,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:58,314 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:58,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:53:58,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:58,316 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:58,316 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/A is initiating minor compaction (all files) 2024-11-12T00:53:58,316 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/A in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:58,317 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/dc7b0df68777418d8a6efc80e8ea95df, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/a91eccadcbf44278bdbea0511ee3ae40, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/63c4a59f03374dddb1877bebf5624918] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=37.4 K 2024-11-12T00:53:58,317 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:58,317 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/B is initiating minor compaction (all files) 2024-11-12T00:53:58,317 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/B in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:58,317 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/14054e439a0f4e379217252df1aea48c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/7867a159f5b541808f69061f296f9680, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/e17e292b467a49a3b649076ad28bbe48] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=35.2 K 2024-11-12T00:53:58,317 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc7b0df68777418d8a6efc80e8ea95df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731372836030 2024-11-12T00:53:58,318 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting a91eccadcbf44278bdbea0511ee3ae40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731372836063 2024-11-12T00:53:58,318 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 14054e439a0f4e379217252df1aea48c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731372836030 2024-11-12T00:53:58,318 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63c4a59f03374dddb1877bebf5624918, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731372836697 2024-11-12T00:53:58,319 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 7867a159f5b541808f69061f296f9680, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731372836063 2024-11-12T00:53:58,319 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting e17e292b467a49a3b649076ad28bbe48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731372836697 2024-11-12T00:53:58,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-12T00:53:58,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:58,323 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T00:53:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:53:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:53:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:53:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:58,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/6c8781923b444095bcd31f35adcf17e5 is 50, key is test_row_0/A:col10/1731372837866/Put/seqid=0 2024-11-12T00:53:58,331 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#A#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:58,331 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/b3ef2c7a0cf949e89a99a19556d7d9d2 is 50, key is test_row_0/A:col10/1731372836697/Put/seqid=0 2024-11-12T00:53:58,334 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#B#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:58,334 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/9055ecae73604821ade088b9cf3fe9fd is 50, key is test_row_0/B:col10/1731372836697/Put/seqid=0 2024-11-12T00:53:58,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742077_1253 (size=12001) 2024-11-12T00:53:58,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742078_1254 (size=12104) 2024-11-12T00:53:58,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742079_1255 (size=12104) 2024-11-12T00:53:58,367 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/b3ef2c7a0cf949e89a99a19556d7d9d2 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b3ef2c7a0cf949e89a99a19556d7d9d2 2024-11-12T00:53:58,370 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/9055ecae73604821ade088b9cf3fe9fd as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/9055ecae73604821ade088b9cf3fe9fd 2024-11-12T00:53:58,374 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/A of eacf29d404b7a11371b9ac1286c14c2b into b3ef2c7a0cf949e89a99a19556d7d9d2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:58,374 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:53:58,374 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/A, priority=13, startTime=1731372838314; duration=0sec 2024-11-12T00:53:58,374 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:53:58,375 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:A 2024-11-12T00:53:58,375 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:53:58,380 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/B of eacf29d404b7a11371b9ac1286c14c2b into 9055ecae73604821ade088b9cf3fe9fd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:58,380 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:53:58,380 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/B, priority=13, startTime=1731372838314; duration=0sec 2024-11-12T00:53:58,380 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:58,380 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:B 2024-11-12T00:53:58,381 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:53:58,381 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/C is initiating minor compaction (all files) 2024-11-12T00:53:58,381 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/C in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:58,381 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/437487d892614748b9e216b082e90631, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/1df66fa9b90b4e61966f91c30a328e48, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9a478915c1184145831919ab233b83dc] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=35.2 K 2024-11-12T00:53:58,382 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 437487d892614748b9e216b082e90631, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731372836030 2024-11-12T00:53:58,384 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1df66fa9b90b4e61966f91c30a328e48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731372836063 2024-11-12T00:53:58,384 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a478915c1184145831919ab233b83dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731372836697 2024-11-12T00:53:58,393 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#C#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:53:58,394 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/c858830a30584f70b8c2ba2152e0d4d9 is 50, key is test_row_0/C:col10/1731372836697/Put/seqid=0 2024-11-12T00:53:58,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742080_1256 (size=12104) 2024-11-12T00:53:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T00:53:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:58,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:53:58,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372898516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372898523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372898523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372898523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372898523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372898625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372898636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372898636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372898636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372898636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,746 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/6c8781923b444095bcd31f35adcf17e5 2024-11-12T00:53:58,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/016e4e7960d546a4bd26eb1739f1d9f3 is 50, key is test_row_0/B:col10/1731372837866/Put/seqid=0 2024-11-12T00:53:58,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742081_1257 (size=12001) 2024-11-12T00:53:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T00:53:58,832 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/c858830a30584f70b8c2ba2152e0d4d9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/c858830a30584f70b8c2ba2152e0d4d9 2024-11-12T00:53:58,838 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/C of eacf29d404b7a11371b9ac1286c14c2b into c858830a30584f70b8c2ba2152e0d4d9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:53:58,838 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:53:58,838 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/C, priority=13, startTime=1731372838314; duration=0sec 2024-11-12T00:53:58,838 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:53:58,838 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:C 2024-11-12T00:53:58,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372898836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372898846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372898846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372898846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:58,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372898846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:58,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:58,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,020 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T00:53:59,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:59,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372899145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:59,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372899156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:59,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:59,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372899156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:59,158 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/016e4e7960d546a4bd26eb1739f1d9f3 2024-11-12T00:53:59,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:59,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372899156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:59,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372899156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/016986bd53584419a7f29c20bdd6175c is 50, key is test_row_0/C:col10/1731372837866/Put/seqid=0 2024-11-12T00:53:59,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742082_1258 (size=12001) 2024-11-12T00:53:59,204 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/016986bd53584419a7f29c20bdd6175c 2024-11-12T00:53:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/6c8781923b444095bcd31f35adcf17e5 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/6c8781923b444095bcd31f35adcf17e5 2024-11-12T00:53:59,215 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/6c8781923b444095bcd31f35adcf17e5, entries=150, sequenceid=66, filesize=11.7 K 2024-11-12T00:53:59,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/016e4e7960d546a4bd26eb1739f1d9f3 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/016e4e7960d546a4bd26eb1739f1d9f3 2024-11-12T00:53:59,222 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/016e4e7960d546a4bd26eb1739f1d9f3, entries=150, sequenceid=66, filesize=11.7 K 2024-11-12T00:53:59,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/016986bd53584419a7f29c20bdd6175c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/016986bd53584419a7f29c20bdd6175c 2024-11-12T00:53:59,227 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/016986bd53584419a7f29c20bdd6175c, entries=150, sequenceid=66, filesize=11.7 K 2024-11-12T00:53:59,228 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for eacf29d404b7a11371b9ac1286c14c2b in 906ms, sequenceid=66, compaction requested=false 2024-11-12T00:53:59,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:53:59,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:59,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-12T00:53:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-12T00:53:59,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-12T00:53:59,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0630 sec 2024-11-12T00:53:59,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.0680 sec 2024-11-12T00:53:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T00:53:59,306 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:59,306 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-12T00:53:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T00:53:59,307 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:59,308 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:59,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T00:53:59,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-12T00:53:59,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:59,460 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T00:53:59,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:53:59,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:59,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:53:59,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:59,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:53:59,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:59,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/81fc997e00354e3495c8646abf384a75 is 50, key is test_row_0/A:col10/1731372838519/Put/seqid=0 2024-11-12T00:53:59,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742083_1259 (size=12001) 2024-11-12T00:53:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T00:53:59,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:59,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:53:59,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:59,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372899676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:59,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372899676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:59,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372899676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:59,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372899679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:59,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372899680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372899785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:59,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372899785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372899785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:59,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372899786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:59,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:53:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372899796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:53:59,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:53:59,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:53:59,871 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/81fc997e00354e3495c8646abf384a75 2024-11-12T00:53:59,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/e95ff23636ec484599897a22cd63f7c3 is 50, key is test_row_0/B:col10/1731372838519/Put/seqid=0 2024-11-12T00:53:59,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742084_1260 (size=12001) 2024-11-12T00:53:59,882 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/e95ff23636ec484599897a22cd63f7c3 2024-11-12T00:53:59,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/677a8b7483144770bd8e15ba95b8310a is 50, key is test_row_0/C:col10/1731372838519/Put/seqid=0 2024-11-12T00:53:59,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742085_1261 (size=12001) 2024-11-12T00:53:59,899 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/677a8b7483144770bd8e15ba95b8310a 2024-11-12T00:53:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/81fc997e00354e3495c8646abf384a75 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/81fc997e00354e3495c8646abf384a75 2024-11-12T00:53:59,906 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/81fc997e00354e3495c8646abf384a75, entries=150, sequenceid=85, filesize=11.7 K 2024-11-12T00:53:59,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/e95ff23636ec484599897a22cd63f7c3 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/e95ff23636ec484599897a22cd63f7c3 2024-11-12T00:53:59,911 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/e95ff23636ec484599897a22cd63f7c3, entries=150, sequenceid=85, filesize=11.7 K 2024-11-12T00:53:59,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/677a8b7483144770bd8e15ba95b8310a as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/677a8b7483144770bd8e15ba95b8310a 2024-11-12T00:53:59,916 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/677a8b7483144770bd8e15ba95b8310a, entries=150, sequenceid=85, filesize=11.7 K 2024-11-12T00:53:59,918 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for eacf29d404b7a11371b9ac1286c14c2b in 458ms, sequenceid=85, compaction requested=true 2024-11-12T00:53:59,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:53:59,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:53:59,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-12T00:53:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-12T00:53:59,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-12T00:53:59,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 611 msec 2024-11-12T00:53:59,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 615 msec 2024-11-12T00:53:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T00:53:59,935 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:53:59,936 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:53:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-12T00:53:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T00:53:59,939 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:53:59,940 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:53:59,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:53:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:53:59,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T00:53:59,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:53:59,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:59,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:53:59,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:53:59,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:53:59,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:00,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/7667e257b8ef428d95ff52464c164cb4 is 50, key is test_row_0/A:col10/1731372839997/Put/seqid=0 2024-11-12T00:54:00,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742086_1262 (size=12001) 2024-11-12T00:54:00,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372900010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372900011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372900012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372900012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372900013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T00:54:00,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T00:54:00,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:00,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:00,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:00,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372900115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372900125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372900126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372900126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372900126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T00:54:00,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:00,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:00,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:00,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T00:54:00,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372900326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372900335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372900335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372900335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372900335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T00:54:00,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:00,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:00,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:00,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/7667e257b8ef428d95ff52464c164cb4 2024-11-12T00:54:00,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/5178d0f69ebe404c9429cdff909c9ed3 is 50, key is test_row_0/B:col10/1731372839997/Put/seqid=0 2024-11-12T00:54:00,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742087_1263 (size=12001) 2024-11-12T00:54:00,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T00:54:00,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:00,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:00,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:00,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T00:54:00,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372900636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372900645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372900645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372900645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372900645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:00,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:00,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T00:54:00,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:00,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:00,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:00,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/5178d0f69ebe404c9429cdff909c9ed3 2024-11-12T00:54:00,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/4ce2f51f77584ff3b8e27c52ca6e9296 is 50, key is test_row_0/C:col10/1731372839997/Put/seqid=0 2024-11-12T00:54:00,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742088_1264 (size=12001) 2024-11-12T00:54:00,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/4ce2f51f77584ff3b8e27c52ca6e9296 2024-11-12T00:54:00,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/7667e257b8ef428d95ff52464c164cb4 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7667e257b8ef428d95ff52464c164cb4 2024-11-12T00:54:00,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T00:54:00,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:00,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:00,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:00,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:00,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7667e257b8ef428d95ff52464c164cb4, entries=150, sequenceid=103, filesize=11.7 K 2024-11-12T00:54:00,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/5178d0f69ebe404c9429cdff909c9ed3 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/5178d0f69ebe404c9429cdff909c9ed3 2024-11-12T00:54:00,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/5178d0f69ebe404c9429cdff909c9ed3, entries=150, sequenceid=103, filesize=11.7 K 2024-11-12T00:54:00,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/4ce2f51f77584ff3b8e27c52ca6e9296 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4ce2f51f77584ff3b8e27c52ca6e9296 2024-11-12T00:54:00,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4ce2f51f77584ff3b8e27c52ca6e9296, entries=150, sequenceid=103, filesize=11.7 K 2024-11-12T00:54:00,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for eacf29d404b7a11371b9ac1286c14c2b in 875ms, sequenceid=103, compaction requested=true 2024-11-12T00:54:00,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:00,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:54:00,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:00,873 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:54:00,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:54:00,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:00,874 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:54:00,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:54:00,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:00,875 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:54:00,875 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:54:00,875 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/A is initiating minor compaction (all files) 2024-11-12T00:54:00,875 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/B is initiating minor compaction (all files) 2024-11-12T00:54:00,875 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/A in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:00,875 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/B in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:00,875 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b3ef2c7a0cf949e89a99a19556d7d9d2, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/6c8781923b444095bcd31f35adcf17e5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/81fc997e00354e3495c8646abf384a75, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7667e257b8ef428d95ff52464c164cb4] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=47.0 K 2024-11-12T00:54:00,875 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/9055ecae73604821ade088b9cf3fe9fd, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/016e4e7960d546a4bd26eb1739f1d9f3, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/e95ff23636ec484599897a22cd63f7c3, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/5178d0f69ebe404c9429cdff909c9ed3] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=47.0 K 2024-11-12T00:54:00,875 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3ef2c7a0cf949e89a99a19556d7d9d2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731372836697 2024-11-12T00:54:00,875 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 9055ecae73604821ade088b9cf3fe9fd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731372836697 2024-11-12T00:54:00,876 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c8781923b444095bcd31f35adcf17e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731372837855 2024-11-12T00:54:00,876 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 016e4e7960d546a4bd26eb1739f1d9f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731372837855 2024-11-12T00:54:00,876 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81fc997e00354e3495c8646abf384a75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731372838519 2024-11-12T00:54:00,876 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting e95ff23636ec484599897a22cd63f7c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731372838519 2024-11-12T00:54:00,876 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7667e257b8ef428d95ff52464c164cb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731372839674 2024-11-12T00:54:00,876 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 5178d0f69ebe404c9429cdff909c9ed3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731372839674 2024-11-12T00:54:00,885 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#B#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:00,885 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#A#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:00,885 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/67753333e3b24f6882e9949cc4d5d33c is 50, key is test_row_0/B:col10/1731372839997/Put/seqid=0 2024-11-12T00:54:00,886 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/b0b0b01cfebf42648aa349162b3602a9 is 50, key is test_row_0/A:col10/1731372839997/Put/seqid=0 2024-11-12T00:54:00,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742090_1266 (size=12241) 2024-11-12T00:54:00,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742089_1265 (size=12241) 2024-11-12T00:54:00,903 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/67753333e3b24f6882e9949cc4d5d33c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/67753333e3b24f6882e9949cc4d5d33c 2024-11-12T00:54:00,908 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/B of eacf29d404b7a11371b9ac1286c14c2b into 67753333e3b24f6882e9949cc4d5d33c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:00,908 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:00,908 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/B, priority=12, startTime=1731372840873; duration=0sec 2024-11-12T00:54:00,908 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:00,908 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:B 2024-11-12T00:54:00,908 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:54:00,909 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:54:00,909 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/C is initiating minor compaction (all files) 2024-11-12T00:54:00,909 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/C in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:00,909 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/c858830a30584f70b8c2ba2152e0d4d9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/016986bd53584419a7f29c20bdd6175c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/677a8b7483144770bd8e15ba95b8310a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4ce2f51f77584ff3b8e27c52ca6e9296] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=47.0 K 2024-11-12T00:54:00,910 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting c858830a30584f70b8c2ba2152e0d4d9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731372836697 2024-11-12T00:54:00,910 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 016986bd53584419a7f29c20bdd6175c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731372837855 2024-11-12T00:54:00,910 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 677a8b7483144770bd8e15ba95b8310a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731372838519 2024-11-12T00:54:00,911 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce2f51f77584ff3b8e27c52ca6e9296, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731372839674 2024-11-12T00:54:00,920 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:00,920 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/b0c48449177d4f9faaee82e7d9d6cde6 is 50, key is test_row_0/C:col10/1731372839997/Put/seqid=0 2024-11-12T00:54:00,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742091_1267 (size=12241) 2024-11-12T00:54:01,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T00:54:01,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:01,012 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T00:54:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:01,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:01,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/5ff0ff789cbc433ab50d7158b8b4ead4 is 50, key is test_row_0/A:col10/1731372840011/Put/seqid=0 2024-11-12T00:54:01,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742092_1268 (size=12001) 2024-11-12T00:54:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T00:54:01,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:01,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372901173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372901174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372901177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372901178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372901178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372901286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372901286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372901286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372901286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372901286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,302 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/b0b0b01cfebf42648aa349162b3602a9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b0b0b01cfebf42648aa349162b3602a9 2024-11-12T00:54:01,306 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/A of eacf29d404b7a11371b9ac1286c14c2b into b0b0b01cfebf42648aa349162b3602a9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:01,306 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:01,306 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/A, priority=12, startTime=1731372840873; duration=0sec 2024-11-12T00:54:01,306 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:01,306 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:A 2024-11-12T00:54:01,329 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/b0c48449177d4f9faaee82e7d9d6cde6 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/b0c48449177d4f9faaee82e7d9d6cde6 2024-11-12T00:54:01,334 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/C of eacf29d404b7a11371b9ac1286c14c2b into b0c48449177d4f9faaee82e7d9d6cde6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:01,334 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:01,334 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/C, priority=12, startTime=1731372840874; duration=0sec 2024-11-12T00:54:01,335 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:01,335 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:C 2024-11-12T00:54:01,441 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/5ff0ff789cbc433ab50d7158b8b4ead4 2024-11-12T00:54:01,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/857e6c91b7c54d1dbc9e4431fa058934 is 50, key is test_row_0/B:col10/1731372840011/Put/seqid=0 2024-11-12T00:54:01,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742093_1269 (size=12001) 2024-11-12T00:54:01,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372901496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372901496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372901496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372901496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372901496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372901805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372901805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372901805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372901806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372901806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:01,851 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/857e6c91b7c54d1dbc9e4431fa058934 2024-11-12T00:54:01,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/4d3259e61cdd479ba1a519fd7ab872d6 is 50, key is test_row_0/C:col10/1731372840011/Put/seqid=0 2024-11-12T00:54:01,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742094_1270 (size=12001) 2024-11-12T00:54:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T00:54:02,262 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/4d3259e61cdd479ba1a519fd7ab872d6 2024-11-12T00:54:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/5ff0ff789cbc433ab50d7158b8b4ead4 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/5ff0ff789cbc433ab50d7158b8b4ead4 2024-11-12T00:54:02,270 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/5ff0ff789cbc433ab50d7158b8b4ead4, entries=150, sequenceid=118, filesize=11.7 K 2024-11-12T00:54:02,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/857e6c91b7c54d1dbc9e4431fa058934 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/857e6c91b7c54d1dbc9e4431fa058934 2024-11-12T00:54:02,276 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/857e6c91b7c54d1dbc9e4431fa058934, entries=150, sequenceid=118, filesize=11.7 K 2024-11-12T00:54:02,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/4d3259e61cdd479ba1a519fd7ab872d6 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4d3259e61cdd479ba1a519fd7ab872d6 2024-11-12T00:54:02,287 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4d3259e61cdd479ba1a519fd7ab872d6, entries=150, sequenceid=118, filesize=11.7 K 2024-11-12T00:54:02,288 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for eacf29d404b7a11371b9ac1286c14c2b in 1277ms, sequenceid=118, compaction requested=false 2024-11-12T00:54:02,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:02,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:02,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-12T00:54:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-12T00:54:02,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-12T00:54:02,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3500 sec 2024-11-12T00:54:02,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.3610 sec 2024-11-12T00:54:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:02,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T00:54:02,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:02,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:02,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:02,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:02,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:02,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:02,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/a2c6c1198a6b44b1bdc64bff121df735 is 50, key is test_row_0/A:col10/1731372841175/Put/seqid=0 2024-11-12T00:54:02,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742095_1271 (size=12151) 2024-11-12T00:54:02,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372902331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372902332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372902333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372902337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372902338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372902446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372902446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372902446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372902456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372902456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372902656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372902656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372902656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372902665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372902665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/a2c6c1198a6b44b1bdc64bff121df735 2024-11-12T00:54:02,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/1d1cc9ea7c154c89868aa3a39c05fdb7 is 50, key is test_row_0/B:col10/1731372841175/Put/seqid=0 2024-11-12T00:54:02,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742096_1272 (size=12151) 2024-11-12T00:54:02,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372902965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372902966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372902966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372902975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372902975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:03,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/1d1cc9ea7c154c89868aa3a39c05fdb7 2024-11-12T00:54:03,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/6ee5687a2a3f47e290341bb880af6fc1 is 50, key is test_row_0/C:col10/1731372841175/Put/seqid=0 2024-11-12T00:54:03,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742097_1273 (size=12151) 2024-11-12T00:54:03,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372903475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:03,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372903475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:03,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372903475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:03,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372903486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:03,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372903486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:03,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:03,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/6ee5687a2a3f47e290341bb880af6fc1 2024-11-12T00:54:03,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/a2c6c1198a6b44b1bdc64bff121df735 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/a2c6c1198a6b44b1bdc64bff121df735 2024-11-12T00:54:03,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/a2c6c1198a6b44b1bdc64bff121df735, entries=150, sequenceid=139, filesize=11.9 K 2024-11-12T00:54:03,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/1d1cc9ea7c154c89868aa3a39c05fdb7 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/1d1cc9ea7c154c89868aa3a39c05fdb7 2024-11-12T00:54:03,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/1d1cc9ea7c154c89868aa3a39c05fdb7, entries=150, sequenceid=139, filesize=11.9 K 2024-11-12T00:54:03,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/6ee5687a2a3f47e290341bb880af6fc1 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6ee5687a2a3f47e290341bb880af6fc1 2024-11-12T00:54:03,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6ee5687a2a3f47e290341bb880af6fc1, entries=150, sequenceid=139, filesize=11.9 K 2024-11-12T00:54:03,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for eacf29d404b7a11371b9ac1286c14c2b in 1250ms, sequenceid=139, compaction requested=true 2024-11-12T00:54:03,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:03,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:54:03,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:03,569 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:03,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:54:03,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:03,569 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:03,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:54:03,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:03,570 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:03,570 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:03,570 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/B is initiating minor compaction (all files) 2024-11-12T00:54:03,570 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/A is initiating minor compaction (all files) 2024-11-12T00:54:03,570 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/B in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:03,570 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/A in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:03,570 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b0b0b01cfebf42648aa349162b3602a9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/5ff0ff789cbc433ab50d7158b8b4ead4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/a2c6c1198a6b44b1bdc64bff121df735] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=35.5 K 2024-11-12T00:54:03,570 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/67753333e3b24f6882e9949cc4d5d33c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/857e6c91b7c54d1dbc9e4431fa058934, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/1d1cc9ea7c154c89868aa3a39c05fdb7] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=35.5 K 2024-11-12T00:54:03,570 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 67753333e3b24f6882e9949cc4d5d33c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731372839674 2024-11-12T00:54:03,570 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0b0b01cfebf42648aa349162b3602a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731372839674 2024-11-12T00:54:03,571 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 857e6c91b7c54d1dbc9e4431fa058934, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731372840009 2024-11-12T00:54:03,571 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ff0ff789cbc433ab50d7158b8b4ead4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731372840009 2024-11-12T00:54:03,571 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d1cc9ea7c154c89868aa3a39c05fdb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731372841175 2024-11-12T00:54:03,571 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2c6c1198a6b44b1bdc64bff121df735, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731372841175 2024-11-12T00:54:03,578 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#A#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:03,579 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/6529a82fd4a845d5a08a1363e8e9e2c5 is 50, key is test_row_0/A:col10/1731372841175/Put/seqid=0 2024-11-12T00:54:03,581 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:03,582 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/36804d7c27724cc781e5e0171e0ecc9e is 50, key is test_row_0/B:col10/1731372841175/Put/seqid=0 2024-11-12T00:54:03,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742098_1274 (size=12493) 2024-11-12T00:54:03,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742099_1275 (size=12493) 2024-11-12T00:54:03,991 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/6529a82fd4a845d5a08a1363e8e9e2c5 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/6529a82fd4a845d5a08a1363e8e9e2c5 2024-11-12T00:54:03,996 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/A of eacf29d404b7a11371b9ac1286c14c2b into 6529a82fd4a845d5a08a1363e8e9e2c5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:03,996 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:03,996 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/A, priority=13, startTime=1731372843568; duration=0sec 2024-11-12T00:54:03,996 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:03,996 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:A 2024-11-12T00:54:03,996 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:03,997 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:03,997 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/C is initiating minor compaction (all files) 2024-11-12T00:54:03,997 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/C in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:03,997 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/b0c48449177d4f9faaee82e7d9d6cde6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4d3259e61cdd479ba1a519fd7ab872d6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6ee5687a2a3f47e290341bb880af6fc1] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=35.5 K 2024-11-12T00:54:03,998 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0c48449177d4f9faaee82e7d9d6cde6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731372839674 2024-11-12T00:54:03,998 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d3259e61cdd479ba1a519fd7ab872d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731372840009 2024-11-12T00:54:03,999 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ee5687a2a3f47e290341bb880af6fc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731372841175 2024-11-12T00:54:04,000 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/36804d7c27724cc781e5e0171e0ecc9e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/36804d7c27724cc781e5e0171e0ecc9e 2024-11-12T00:54:04,007 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/B of eacf29d404b7a11371b9ac1286c14c2b into 36804d7c27724cc781e5e0171e0ecc9e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:04,007 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:04,007 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/B, priority=13, startTime=1731372843569; duration=0sec 2024-11-12T00:54:04,007 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:04,007 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:B 2024-11-12T00:54:04,007 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#C#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:04,008 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/915667c826384b03a6c6fcc9b562aba8 is 50, key is test_row_0/C:col10/1731372841175/Put/seqid=0 2024-11-12T00:54:04,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742100_1276 (size=12493) 2024-11-12T00:54:04,026 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/915667c826384b03a6c6fcc9b562aba8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/915667c826384b03a6c6fcc9b562aba8 2024-11-12T00:54:04,031 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/C of eacf29d404b7a11371b9ac1286c14c2b into 915667c826384b03a6c6fcc9b562aba8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:04,031 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:04,031 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/C, priority=13, startTime=1731372843569; duration=0sec 2024-11-12T00:54:04,032 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:04,032 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:C 2024-11-12T00:54:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T00:54:04,106 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:04,106 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:54:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-12T00:54:04,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T00:54:04,110 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:54:04,111 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:54:04,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:54:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T00:54:04,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T00:54:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:04,264 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T00:54:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:04,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/b4221fa919994253a0b758781a445f94 is 50, key is test_row_0/A:col10/1731372842335/Put/seqid=0 2024-11-12T00:54:04,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742101_1277 (size=12151) 2024-11-12T00:54:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T00:54:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:04,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:04,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:04,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372904505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:04,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:04,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372904506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:04,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:04,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372904507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:04,510 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in eacf29d404b7a11371b9ac1286c14c2b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:04,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:04,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:04,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372904509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372904510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:04,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372904615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:04,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372904615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:04,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372904626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:04,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372904626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:04,672 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/b4221fa919994253a0b758781a445f94 2024-11-12T00:54:04,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/57c191a984664c3b97a34e64ea00cd9d is 50, key is test_row_0/B:col10/1731372842335/Put/seqid=0 2024-11-12T00:54:04,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742102_1278 (size=12151) 2024-11-12T00:54:04,691 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/57c191a984664c3b97a34e64ea00cd9d 2024-11-12T00:54:04,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/7b7c615b48bc485dbaedc622f4559bab is 50, key is test_row_0/C:col10/1731372842335/Put/seqid=0 2024-11-12T00:54:04,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742103_1279 (size=12151) 2024-11-12T00:54:04,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T00:54:04,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:04,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372904826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:04,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372904826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:04,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:04,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372904835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:04,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:04,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:04,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372904835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:04,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:04,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,102 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/7b7c615b48bc485dbaedc622f4559bab 2024-11-12T00:54:05,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/b4221fa919994253a0b758781a445f94 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b4221fa919994253a0b758781a445f94 2024-11-12T00:54:05,109 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b4221fa919994253a0b758781a445f94, entries=150, sequenceid=155, filesize=11.9 K 2024-11-12T00:54:05,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/57c191a984664c3b97a34e64ea00cd9d as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/57c191a984664c3b97a34e64ea00cd9d 2024-11-12T00:54:05,115 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/57c191a984664c3b97a34e64ea00cd9d, entries=150, sequenceid=155, filesize=11.9 K 2024-11-12T00:54:05,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/7b7c615b48bc485dbaedc622f4559bab as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b7c615b48bc485dbaedc622f4559bab 2024-11-12T00:54:05,121 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b7c615b48bc485dbaedc622f4559bab, entries=150, sequenceid=155, filesize=11.9 K 2024-11-12T00:54:05,122 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for eacf29d404b7a11371b9ac1286c14c2b in 858ms, sequenceid=155, compaction requested=false 2024-11-12T00:54:05,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:05,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:05,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-12T00:54:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-12T00:54:05,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-12T00:54:05,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0120 sec 2024-11-12T00:54:05,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.0190 sec 2024-11-12T00:54:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:05,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-12T00:54:05,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:05,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:05,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:05,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:05,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:05,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:05,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/7b9fbc2a17b84bd59aa56da0b0d20835 is 50, key is test_row_0/A:col10/1731372845138/Put/seqid=0 2024-11-12T00:54:05,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742104_1280 (size=16931) 2024-11-12T00:54:05,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372905167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372905167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372905172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372905172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T00:54:05,246 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:05,247 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:54:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-12T00:54:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T00:54:05,249 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:54:05,249 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:54:05,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:54:05,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372905275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372905275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372905285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372905285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T00:54:05,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T00:54:05,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:05,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:05,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:05,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:05,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:05,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372905485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372905485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372905495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372905495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T00:54:05,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:05,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:05,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:05,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:05,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T00:54:05,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/7b9fbc2a17b84bd59aa56da0b0d20835 2024-11-12T00:54:05,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/cbed5c097f5f49889cd5de7ac71f72f1 is 50, key is test_row_0/B:col10/1731372845138/Put/seqid=0 2024-11-12T00:54:05,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742105_1281 (size=12151) 2024-11-12T00:54:05,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/cbed5c097f5f49889cd5de7ac71f72f1 2024-11-12T00:54:05,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/9afbf2ebab8341e2aa06e3b873f62723 is 50, key is test_row_0/C:col10/1731372845138/Put/seqid=0 2024-11-12T00:54:05,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742106_1282 (size=12151) 2024-11-12T00:54:05,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/9afbf2ebab8341e2aa06e3b873f62723 2024-11-12T00:54:05,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/7b9fbc2a17b84bd59aa56da0b0d20835 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7b9fbc2a17b84bd59aa56da0b0d20835 2024-11-12T00:54:05,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7b9fbc2a17b84bd59aa56da0b0d20835, entries=250, sequenceid=176, filesize=16.5 K 2024-11-12T00:54:05,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/cbed5c097f5f49889cd5de7ac71f72f1 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/cbed5c097f5f49889cd5de7ac71f72f1 2024-11-12T00:54:05,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/cbed5c097f5f49889cd5de7ac71f72f1, entries=150, sequenceid=176, filesize=11.9 K 2024-11-12T00:54:05,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/9afbf2ebab8341e2aa06e3b873f62723 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9afbf2ebab8341e2aa06e3b873f62723 2024-11-12T00:54:05,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9afbf2ebab8341e2aa06e3b873f62723, entries=150, sequenceid=176, filesize=11.9 K 2024-11-12T00:54:05,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for eacf29d404b7a11371b9ac1286c14c2b in 526ms, sequenceid=176, compaction requested=true 2024-11-12T00:54:05,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:05,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:54:05,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:05,684 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:05,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:54:05,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:05,684 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:05,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:54:05,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:05,686 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:05,686 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/B is initiating minor compaction (all files) 2024-11-12T00:54:05,686 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/B in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:05,686 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/36804d7c27724cc781e5e0171e0ecc9e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/57c191a984664c3b97a34e64ea00cd9d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/cbed5c097f5f49889cd5de7ac71f72f1] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=35.9 K 2024-11-12T00:54:05,686 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:05,686 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/A is initiating minor compaction (all files) 2024-11-12T00:54:05,687 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/A in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:05,687 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/6529a82fd4a845d5a08a1363e8e9e2c5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b4221fa919994253a0b758781a445f94, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7b9fbc2a17b84bd59aa56da0b0d20835] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=40.6 K 2024-11-12T00:54:05,688 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 36804d7c27724cc781e5e0171e0ecc9e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731372841175 2024-11-12T00:54:05,689 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6529a82fd4a845d5a08a1363e8e9e2c5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731372841175 2024-11-12T00:54:05,689 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 57c191a984664c3b97a34e64ea00cd9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731372842330 2024-11-12T00:54:05,690 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4221fa919994253a0b758781a445f94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731372842330 2024-11-12T00:54:05,691 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting cbed5c097f5f49889cd5de7ac71f72f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731372844504 2024-11-12T00:54:05,691 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b9fbc2a17b84bd59aa56da0b0d20835, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731372844504 2024-11-12T00:54:05,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T00:54:05,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:05,708 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T00:54:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:05,711 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#B#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:05,712 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/02ed8983038045e08c0409472d8d83ca is 50, key is test_row_0/B:col10/1731372845138/Put/seqid=0 2024-11-12T00:54:05,715 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#A#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:05,715 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/cd4df67709d1402a9151b8156620f2b7 is 50, key is test_row_0/A:col10/1731372845138/Put/seqid=0 2024-11-12T00:54:05,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/8692685d9acf4d8e83a185f59144e706 is 50, key is test_row_0/A:col10/1731372845171/Put/seqid=0 2024-11-12T00:54:05,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742107_1283 (size=12595) 2024-11-12T00:54:05,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742108_1284 (size=12595) 2024-11-12T00:54:05,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742109_1285 (size=12151) 2024-11-12T00:54:05,754 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/8692685d9acf4d8e83a185f59144e706 2024-11-12T00:54:05,760 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/02ed8983038045e08c0409472d8d83ca as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/02ed8983038045e08c0409472d8d83ca 2024-11-12T00:54:05,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/0da2ce6aa1dd4450a0f334cd1739b103 is 50, key is test_row_0/B:col10/1731372845171/Put/seqid=0 2024-11-12T00:54:05,768 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/B of eacf29d404b7a11371b9ac1286c14c2b into 02ed8983038045e08c0409472d8d83ca(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:05,768 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:05,768 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/B, priority=13, startTime=1731372845684; duration=0sec 2024-11-12T00:54:05,768 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:05,768 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:B 2024-11-12T00:54:05,768 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:05,770 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:05,770 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/C is initiating minor compaction (all files) 2024-11-12T00:54:05,770 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/C in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:05,770 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/915667c826384b03a6c6fcc9b562aba8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b7c615b48bc485dbaedc622f4559bab, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9afbf2ebab8341e2aa06e3b873f62723] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=35.9 K 2024-11-12T00:54:05,771 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 915667c826384b03a6c6fcc9b562aba8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731372841175 2024-11-12T00:54:05,771 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b7c615b48bc485dbaedc622f4559bab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731372842330 2024-11-12T00:54:05,771 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 9afbf2ebab8341e2aa06e3b873f62723, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731372844504 2024-11-12T00:54:05,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742110_1286 (size=12151) 2024-11-12T00:54:05,781 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/0da2ce6aa1dd4450a0f334cd1739b103 2024-11-12T00:54:05,791 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#C#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:05,792 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/6da33c2ac18646deb96e8d530cf8aaa1 is 50, key is test_row_0/C:col10/1731372845138/Put/seqid=0 2024-11-12T00:54:05,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/3bd5a429ba5a4ab886b05db7d14c5332 is 50, key is test_row_0/C:col10/1731372845171/Put/seqid=0 2024-11-12T00:54:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:05,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:05,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742112_1288 (size=12595) 2024-11-12T00:54:05,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742111_1287 (size=12151) 2024-11-12T00:54:05,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731372845847, max timestamp=1731372845855Num uniques 859; 2024-11-12T00:54:05,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T00:54:05,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731372845847, max timestamp=1731372845855Num uniques 859; 2024-11-12T00:54:05,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T00:54:05,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731372845847, max timestamp=1731372845855Num uniques 859; 2024-11-12T00:54:05,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T00:54:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T00:54:05,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372905879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372905879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372905880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372905886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372905995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372905995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372905995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372905996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,152 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/cd4df67709d1402a9151b8156620f2b7 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/cd4df67709d1402a9151b8156620f2b7 2024-11-12T00:54:06,158 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/A of eacf29d404b7a11371b9ac1286c14c2b into cd4df67709d1402a9151b8156620f2b7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:06,158 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:06,158 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/A, priority=13, startTime=1731372845684; duration=0sec 2024-11-12T00:54:06,158 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:06,158 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:A 2024-11-12T00:54:06,206 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/3bd5a429ba5a4ab886b05db7d14c5332 2024-11-12T00:54:06,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372906205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372906206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372906206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372906206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,211 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/6da33c2ac18646deb96e8d530cf8aaa1 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6da33c2ac18646deb96e8d530cf8aaa1 2024-11-12T00:54:06,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/8692685d9acf4d8e83a185f59144e706 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8692685d9acf4d8e83a185f59144e706 2024-11-12T00:54:06,214 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8692685d9acf4d8e83a185f59144e706, entries=150, sequenceid=187, filesize=11.9 K 2024-11-12T00:54:06,215 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/C of eacf29d404b7a11371b9ac1286c14c2b into 6da33c2ac18646deb96e8d530cf8aaa1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:06,215 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:06,215 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/C, priority=13, startTime=1731372845684; duration=0sec 2024-11-12T00:54:06,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/0da2ce6aa1dd4450a0f334cd1739b103 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/0da2ce6aa1dd4450a0f334cd1739b103 2024-11-12T00:54:06,215 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:06,215 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:C 2024-11-12T00:54:06,219 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/0da2ce6aa1dd4450a0f334cd1739b103, entries=150, sequenceid=187, filesize=11.9 K 2024-11-12T00:54:06,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/3bd5a429ba5a4ab886b05db7d14c5332 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3bd5a429ba5a4ab886b05db7d14c5332 2024-11-12T00:54:06,223 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3bd5a429ba5a4ab886b05db7d14c5332, entries=150, sequenceid=187, filesize=11.9 K 2024-11-12T00:54:06,224 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for eacf29d404b7a11371b9ac1286c14c2b in 516ms, sequenceid=187, compaction requested=false 2024-11-12T00:54:06,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:06,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:06,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-12T00:54:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-12T00:54:06,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-12T00:54:06,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 976 msec 2024-11-12T00:54:06,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 981 msec 2024-11-12T00:54:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T00:54:06,385 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:06,386 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:54:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-12T00:54:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T00:54:06,388 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:54:06,388 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:54:06,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:54:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T00:54:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:06,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-12T00:54:06,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:06,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:06,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:06,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:06,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:06,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:06,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/049880fa40bf40779bf7450dc1e6c22d is 50, key is test_row_0/A:col10/1731372846516/Put/seqid=0 2024-11-12T00:54:06,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742113_1289 (size=14541) 2024-11-12T00:54:06,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372906525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372906525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372906526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372906527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,532 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in eacf29d404b7a11371b9ac1286c14c2b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372906528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T00:54:06,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:06,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:06,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:06,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:06,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:06,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372906635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372906635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372906635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372906635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T00:54:06,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:06,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:06,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:06,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:06,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T00:54:06,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T00:54:06,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:06,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:06,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:06,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:06,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:06,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372906845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372906845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372906845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372906845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:06,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/049880fa40bf40779bf7450dc1e6c22d 2024-11-12T00:54:06,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/a24db3d27e874361a1abdcea3fd03780 is 50, key is test_row_0/B:col10/1731372846516/Put/seqid=0 2024-11-12T00:54:06,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742114_1290 (size=12151) 2024-11-12T00:54:06,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T00:54:06,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:06,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:06,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:06,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:06,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T00:54:07,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T00:54:07,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:07,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:07,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:07,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372907156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:07,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372907156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:07,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:07,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372907156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372907156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:07,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T00:54:07,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:07,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:07,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:07,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/a24db3d27e874361a1abdcea3fd03780 2024-11-12T00:54:07,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/7b6135eee5034f7abf8e9bce600aee45 is 50, key is test_row_0/C:col10/1731372846516/Put/seqid=0 2024-11-12T00:54:07,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742115_1291 (size=12151) 2024-11-12T00:54:07,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T00:54:07,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:07,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:07,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:07,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T00:54:07,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T00:54:07,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:07,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:07,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:07,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372907665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:07,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:07,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372907665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372907665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:07,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372907675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:07,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/7b6135eee5034f7abf8e9bce600aee45 2024-11-12T00:54:07,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/049880fa40bf40779bf7450dc1e6c22d as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/049880fa40bf40779bf7450dc1e6c22d 2024-11-12T00:54:07,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/049880fa40bf40779bf7450dc1e6c22d, entries=200, sequenceid=225, filesize=14.2 K 2024-11-12T00:54:07,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/a24db3d27e874361a1abdcea3fd03780 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a24db3d27e874361a1abdcea3fd03780 2024-11-12T00:54:07,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a24db3d27e874361a1abdcea3fd03780, entries=150, sequenceid=225, filesize=11.9 K 2024-11-12T00:54:07,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/7b6135eee5034f7abf8e9bce600aee45 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b6135eee5034f7abf8e9bce600aee45 2024-11-12T00:54:07,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T00:54:07,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:07,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:07,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:07,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:07,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b6135eee5034f7abf8e9bce600aee45, entries=150, sequenceid=225, filesize=11.9 K 2024-11-12T00:54:07,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for eacf29d404b7a11371b9ac1286c14c2b in 1246ms, sequenceid=225, compaction requested=true 2024-11-12T00:54:07,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:07,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:54:07,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:07,764 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:07,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:54:07,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:07,764 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:07,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:54:07,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:07,765 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:07,765 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:07,765 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/A is initiating minor compaction (all files) 2024-11-12T00:54:07,765 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/B is initiating minor compaction (all files) 2024-11-12T00:54:07,765 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/B in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:07,765 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/A in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:07,765 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/02ed8983038045e08c0409472d8d83ca, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/0da2ce6aa1dd4450a0f334cd1739b103, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a24db3d27e874361a1abdcea3fd03780] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=36.0 K 2024-11-12T00:54:07,765 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/cd4df67709d1402a9151b8156620f2b7, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8692685d9acf4d8e83a185f59144e706, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/049880fa40bf40779bf7450dc1e6c22d] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=38.4 K 2024-11-12T00:54:07,766 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 02ed8983038045e08c0409472d8d83ca, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731372844504 2024-11-12T00:54:07,766 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd4df67709d1402a9151b8156620f2b7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731372844504 2024-11-12T00:54:07,766 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8692685d9acf4d8e83a185f59144e706, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731372845165 2024-11-12T00:54:07,766 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 0da2ce6aa1dd4450a0f334cd1739b103, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731372845165 2024-11-12T00:54:07,766 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 049880fa40bf40779bf7450dc1e6c22d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731372845871 2024-11-12T00:54:07,766 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting a24db3d27e874361a1abdcea3fd03780, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731372845878 2024-11-12T00:54:07,774 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:07,775 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/7cff125bb0234640b13080906355b854 is 50, key is test_row_0/A:col10/1731372846516/Put/seqid=0 2024-11-12T00:54:07,780 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#B#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:07,781 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/263242d6d0dc44028e16200caba332eb is 50, key is test_row_0/B:col10/1731372846516/Put/seqid=0 2024-11-12T00:54:07,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742117_1293 (size=12697) 2024-11-12T00:54:07,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742116_1292 (size=12697) 2024-11-12T00:54:07,819 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/7cff125bb0234640b13080906355b854 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7cff125bb0234640b13080906355b854 2024-11-12T00:54:07,824 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/A of eacf29d404b7a11371b9ac1286c14c2b into 7cff125bb0234640b13080906355b854(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:07,824 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:07,824 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/A, priority=13, startTime=1731372847764; duration=0sec 2024-11-12T00:54:07,824 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:07,824 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:A 2024-11-12T00:54:07,824 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:07,825 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:07,825 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/C is initiating minor compaction (all files) 2024-11-12T00:54:07,825 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/C in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:07,825 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6da33c2ac18646deb96e8d530cf8aaa1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3bd5a429ba5a4ab886b05db7d14c5332, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b6135eee5034f7abf8e9bce600aee45] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=36.0 K 2024-11-12T00:54:07,825 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6da33c2ac18646deb96e8d530cf8aaa1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731372844504 2024-11-12T00:54:07,826 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bd5a429ba5a4ab886b05db7d14c5332, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731372845165 2024-11-12T00:54:07,826 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b6135eee5034f7abf8e9bce600aee45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731372845878 2024-11-12T00:54:07,833 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#C#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:07,833 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/f82ad5be7eee4d669d62b72ccd252ed9 is 50, key is test_row_0/C:col10/1731372846516/Put/seqid=0 2024-11-12T00:54:07,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742118_1294 (size=12697) 2024-11-12T00:54:07,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T00:54:07,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:07,915 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T00:54:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:07,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/bb1928b77c3d4c8c93026549187d7b1e is 50, key is test_row_0/A:col10/1731372846527/Put/seqid=0 2024-11-12T00:54:07,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742119_1295 (size=12151) 2024-11-12T00:54:08,219 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/263242d6d0dc44028e16200caba332eb as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/263242d6d0dc44028e16200caba332eb 2024-11-12T00:54:08,223 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/B of eacf29d404b7a11371b9ac1286c14c2b into 263242d6d0dc44028e16200caba332eb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:08,223 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:08,223 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/B, priority=13, startTime=1731372847764; duration=0sec 2024-11-12T00:54:08,223 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:08,223 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:B 2024-11-12T00:54:08,241 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/f82ad5be7eee4d669d62b72ccd252ed9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/f82ad5be7eee4d669d62b72ccd252ed9 2024-11-12T00:54:08,246 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/C of eacf29d404b7a11371b9ac1286c14c2b into f82ad5be7eee4d669d62b72ccd252ed9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:08,246 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:08,246 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/C, priority=13, startTime=1731372847764; duration=0sec 2024-11-12T00:54:08,246 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:08,246 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:C 2024-11-12T00:54:08,354 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/bb1928b77c3d4c8c93026549187d7b1e 2024-11-12T00:54:08,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/aee380c5d3bd47deb8c4169cb0d8dbfe is 50, key is test_row_0/B:col10/1731372846527/Put/seqid=0 2024-11-12T00:54:08,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742120_1296 (size=12151) 2024-11-12T00:54:08,368 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/aee380c5d3bd47deb8c4169cb0d8dbfe 2024-11-12T00:54:08,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/3a5afbfe1d7f458784781c6da8c4f095 is 50, key is test_row_0/C:col10/1731372846527/Put/seqid=0 2024-11-12T00:54:08,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742121_1297 (size=12151) 2024-11-12T00:54:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T00:54:08,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:08,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:08,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372848695, max timestamp=1731372848701Num uniques 859; 2024-11-12T00:54:08,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:54:08,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372848695, max timestamp=1731372848701Num uniques 859; 2024-11-12T00:54:08,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:08,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:54:08,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372848695, max timestamp=1731372848701Num uniques 859; 2024-11-12T00:54:08,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:54:08,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372908700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:08,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372908726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:08,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372908727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:08,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372908731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:08,792 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/3a5afbfe1d7f458784781c6da8c4f095 2024-11-12T00:54:08,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/bb1928b77c3d4c8c93026549187d7b1e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/bb1928b77c3d4c8c93026549187d7b1e 2024-11-12T00:54:08,800 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/bb1928b77c3d4c8c93026549187d7b1e, entries=150, sequenceid=236, filesize=11.9 K 2024-11-12T00:54:08,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/aee380c5d3bd47deb8c4169cb0d8dbfe as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/aee380c5d3bd47deb8c4169cb0d8dbfe 2024-11-12T00:54:08,805 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/aee380c5d3bd47deb8c4169cb0d8dbfe, entries=150, sequenceid=236, filesize=11.9 K 2024-11-12T00:54:08,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/3a5afbfe1d7f458784781c6da8c4f095 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3a5afbfe1d7f458784781c6da8c4f095 2024-11-12T00:54:08,810 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3a5afbfe1d7f458784781c6da8c4f095, entries=150, sequenceid=236, filesize=11.9 K 2024-11-12T00:54:08,811 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for eacf29d404b7a11371b9ac1286c14c2b in 895ms, sequenceid=236, compaction requested=false 2024-11-12T00:54:08,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:08,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:08,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-12T00:54:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-12T00:54:08,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-12T00:54:08,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4240 sec 2024-11-12T00:54:08,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.4290 sec 2024-11-12T00:54:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:08,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-12T00:54:08,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:08,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T00:54:08,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:08,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T00:54:08,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:08,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T00:54:08,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/be1de6f289794b95b7701e47f5e72fbe is 50, key is test_row_0/A:col10/1731372848726/Put/seqid=0 2024-11-12T00:54:08,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742122_1298 (size=12301) 2024-11-12T00:54:08,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/be1de6f289794b95b7701e47f5e72fbe 2024-11-12T00:54:08,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/834c2f8be5df4b7b9fb6c0cef420f917 is 50, key is test_row_0/B:col10/1731372848726/Put/seqid=0 2024-11-12T00:54:08,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742123_1299 (size=12301) 2024-11-12T00:54:08,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372908840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372908841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:08,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372908844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:08,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372908845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:08,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372908955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:08,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372908955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372908956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372909055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372909165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372909165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372909166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/834c2f8be5df4b7b9fb6c0cef420f917 2024-11-12T00:54:09,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/9d70ed88f86843c098be111a9c17b464 is 50, key is test_row_0/C:col10/1731372848726/Put/seqid=0 2024-11-12T00:54:09,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742124_1300 (size=12301) 2024-11-12T00:54:09,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/9d70ed88f86843c098be111a9c17b464 2024-11-12T00:54:09,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/be1de6f289794b95b7701e47f5e72fbe as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/be1de6f289794b95b7701e47f5e72fbe 2024-11-12T00:54:09,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/be1de6f289794b95b7701e47f5e72fbe, entries=150, sequenceid=276, filesize=12.0 K 2024-11-12T00:54:09,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/834c2f8be5df4b7b9fb6c0cef420f917 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/834c2f8be5df4b7b9fb6c0cef420f917 2024-11-12T00:54:09,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/834c2f8be5df4b7b9fb6c0cef420f917, entries=150, sequenceid=276, filesize=12.0 K 2024-11-12T00:54:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/9d70ed88f86843c098be111a9c17b464 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9d70ed88f86843c098be111a9c17b464 2024-11-12T00:54:09,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9d70ed88f86843c098be111a9c17b464, entries=150, sequenceid=276, filesize=12.0 K 2024-11-12T00:54:09,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for eacf29d404b7a11371b9ac1286c14c2b in 472ms, sequenceid=276, compaction requested=true 2024-11-12T00:54:09,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:09,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:54:09,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:09,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:54:09,291 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:09,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:09,291 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:09,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:54:09,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:09,294 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:09,295 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/B is initiating minor compaction (all files) 2024-11-12T00:54:09,295 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/B in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:09,295 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/263242d6d0dc44028e16200caba332eb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/aee380c5d3bd47deb8c4169cb0d8dbfe, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/834c2f8be5df4b7b9fb6c0cef420f917] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=36.3 K 2024-11-12T00:54:09,295 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:09,295 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/A is initiating minor compaction (all files) 2024-11-12T00:54:09,295 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/A in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:09,295 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7cff125bb0234640b13080906355b854, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/bb1928b77c3d4c8c93026549187d7b1e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/be1de6f289794b95b7701e47f5e72fbe] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=36.3 K 2024-11-12T00:54:09,296 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 263242d6d0dc44028e16200caba332eb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731372845878 2024-11-12T00:54:09,296 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cff125bb0234640b13080906355b854, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731372845878 2024-11-12T00:54:09,296 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting aee380c5d3bd47deb8c4169cb0d8dbfe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731372846523 2024-11-12T00:54:09,296 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb1928b77c3d4c8c93026549187d7b1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731372846523 2024-11-12T00:54:09,296 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 834c2f8be5df4b7b9fb6c0cef420f917, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731372848724 2024-11-12T00:54:09,297 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting be1de6f289794b95b7701e47f5e72fbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731372848724 2024-11-12T00:54:09,308 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#A#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:09,308 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/c9531cd40bb84f73879a9fee4348c03a is 50, key is test_row_0/A:col10/1731372848726/Put/seqid=0 2024-11-12T00:54:09,326 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#B#compaction#256 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:09,326 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/225e4392ee1742aba8726ea9c4f14b62 is 50, key is test_row_0/B:col10/1731372848726/Put/seqid=0 2024-11-12T00:54:09,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742126_1302 (size=12949) 2024-11-12T00:54:09,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742125_1301 (size=12949) 2024-11-12T00:54:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:09,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T00:54:09,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:09,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:09,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:09,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:09,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:09,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:09,368 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/225e4392ee1742aba8726ea9c4f14b62 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/225e4392ee1742aba8726ea9c4f14b62 2024-11-12T00:54:09,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/53ab977fac294beea903c8ac4425dacf is 50, key is test_row_0/A:col10/1731372848842/Put/seqid=0 2024-11-12T00:54:09,375 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/B of eacf29d404b7a11371b9ac1286c14c2b into 225e4392ee1742aba8726ea9c4f14b62(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:09,375 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:09,375 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/B, priority=13, startTime=1731372849291; duration=0sec 2024-11-12T00:54:09,375 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:09,375 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:B 2024-11-12T00:54:09,375 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:09,376 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:09,377 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/C is initiating minor compaction (all files) 2024-11-12T00:54:09,377 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/C in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:09,377 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/f82ad5be7eee4d669d62b72ccd252ed9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3a5afbfe1d7f458784781c6da8c4f095, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9d70ed88f86843c098be111a9c17b464] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=36.3 K 2024-11-12T00:54:09,377 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting f82ad5be7eee4d669d62b72ccd252ed9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731372845878 2024-11-12T00:54:09,378 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a5afbfe1d7f458784781c6da8c4f095, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731372846523 2024-11-12T00:54:09,378 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d70ed88f86843c098be111a9c17b464, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731372848724 2024-11-12T00:54:09,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742127_1303 (size=12301) 2024-11-12T00:54:09,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/53ab977fac294beea903c8ac4425dacf 2024-11-12T00:54:09,385 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#C#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:09,386 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/c303672a26ac43ee820eaef305bdfda5 is 50, key is test_row_0/C:col10/1731372848726/Put/seqid=0 2024-11-12T00:54:09,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/f571c88616ab42a28ee760bcc6d31343 is 50, key is test_row_0/B:col10/1731372848842/Put/seqid=0 2024-11-12T00:54:09,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742128_1304 (size=12949) 2024-11-12T00:54:09,419 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/c303672a26ac43ee820eaef305bdfda5 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/c303672a26ac43ee820eaef305bdfda5 2024-11-12T00:54:09,425 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/C of eacf29d404b7a11371b9ac1286c14c2b into c303672a26ac43ee820eaef305bdfda5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:09,425 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:09,425 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/C, priority=13, startTime=1731372849291; duration=0sec 2024-11-12T00:54:09,425 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:09,425 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:C 2024-11-12T00:54:09,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742129_1305 (size=12301) 2024-11-12T00:54:09,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372849450, max timestamp=1731372849469Num uniques 859; 2024-11-12T00:54:09,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:54:09,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372849450, max timestamp=1731372849469Num uniques 859; 2024-11-12T00:54:09,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372849450, max timestamp=1731372849469Num uniques 859; 2024-11-12T00:54:09,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:54:09,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:54:09,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372909499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731372849480, max timestamp=1731372849503Num uniques 1018; 2024-11-12T00:54:09,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T00:54:09,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731372849480, max timestamp=1731372849503Num uniques 1018; 2024-11-12T00:54:09,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731372849480, max timestamp=1731372849503Num uniques 1018; 2024-11-12T00:54:09,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T00:54:09,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T00:54:09,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372909535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372909534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372909538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372909615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372909646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372909646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372909655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,766 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/c9531cd40bb84f73879a9fee4348c03a as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/c9531cd40bb84f73879a9fee4348c03a 2024-11-12T00:54:09,770 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/A of eacf29d404b7a11371b9ac1286c14c2b into c9531cd40bb84f73879a9fee4348c03a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:09,770 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:09,770 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/A, priority=13, startTime=1731372849291; duration=0sec 2024-11-12T00:54:09,770 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:09,770 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:A 2024-11-12T00:54:09,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372909825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/f571c88616ab42a28ee760bcc6d31343 2024-11-12T00:54:09,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/3133167ee5044785896419e442eb82b8 is 50, key is test_row_0/C:col10/1731372848842/Put/seqid=0 2024-11-12T00:54:09,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742130_1306 (size=12301) 2024-11-12T00:54:09,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/3133167ee5044785896419e442eb82b8 2024-11-12T00:54:09,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/53ab977fac294beea903c8ac4425dacf as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/53ab977fac294beea903c8ac4425dacf 2024-11-12T00:54:09,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/53ab977fac294beea903c8ac4425dacf, entries=150, sequenceid=286, filesize=12.0 K 2024-11-12T00:54:09,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/f571c88616ab42a28ee760bcc6d31343 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/f571c88616ab42a28ee760bcc6d31343 2024-11-12T00:54:09,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/f571c88616ab42a28ee760bcc6d31343, entries=150, sequenceid=286, filesize=12.0 K 2024-11-12T00:54:09,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/3133167ee5044785896419e442eb82b8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3133167ee5044785896419e442eb82b8 2024-11-12T00:54:09,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3133167ee5044785896419e442eb82b8, entries=150, sequenceid=286, filesize=12.0 K 2024-11-12T00:54:09,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for eacf29d404b7a11371b9ac1286c14c2b in 490ms, sequenceid=286, compaction requested=false 2024-11-12T00:54:09,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:09,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=140.89 KB heapSize=399.38 KB 2024-11-12T00:54:09,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:09,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:09,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:09,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:09,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:09,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:09,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/219873cc59574275a1e14a4ed4962a5f is 50, key is test_row_0/A:col10/1731372849524/Put/seqid=0 2024-11-12T00:54:09,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742131_1307 (size=12301) 2024-11-12T00:54:09,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372909869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372909870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372909871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372909985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372909985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372909985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:09,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:10,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:10,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372910135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:10,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:10,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372910195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372910195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:10,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372910195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:10,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/219873cc59574275a1e14a4ed4962a5f 2024-11-12T00:54:10,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/5668cd73cf33400691569016686b473a is 50, key is test_row_0/B:col10/1731372849524/Put/seqid=0 2024-11-12T00:54:10,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742132_1308 (size=12301) 2024-11-12T00:54:10,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372910505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:10,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:10,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372910506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:10,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:10,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:10,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372910506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:10,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T00:54:10,556 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:10,556 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:54:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-12T00:54:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T00:54:10,557 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:54:10,558 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:54:10,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:54:10,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39684 deadline: 1731372910565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:10,568 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in eacf29d404b7a11371b9ac1286c14c2b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:10,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372910645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:10,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:10,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T00:54:10,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/5668cd73cf33400691569016686b473a 2024-11-12T00:54:10,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/73fc10e0ba304f4e9b0b140d022fa9e6 is 50, key is test_row_0/C:col10/1731372849524/Put/seqid=0 2024-11-12T00:54:10,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742133_1309 (size=12301) 2024-11-12T00:54:10,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T00:54:10,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:10,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:10,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:10,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:10,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:10,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T00:54:10,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:10,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:10,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:10,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:10,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T00:54:11,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T00:54:11,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:11,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:11,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:11,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:11,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:11,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372911016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:11,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372911017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:11,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:11,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372911017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:11,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/73fc10e0ba304f4e9b0b140d022fa9e6 2024-11-12T00:54:11,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/219873cc59574275a1e14a4ed4962a5f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/219873cc59574275a1e14a4ed4962a5f 2024-11-12T00:54:11,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/219873cc59574275a1e14a4ed4962a5f, entries=150, sequenceid=343, filesize=12.0 K 2024-11-12T00:54:11,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/5668cd73cf33400691569016686b473a as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/5668cd73cf33400691569016686b473a 2024-11-12T00:54:11,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/5668cd73cf33400691569016686b473a, entries=150, sequenceid=343, filesize=12.0 K 2024-11-12T00:54:11,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/73fc10e0ba304f4e9b0b140d022fa9e6 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/73fc10e0ba304f4e9b0b140d022fa9e6 2024-11-12T00:54:11,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/73fc10e0ba304f4e9b0b140d022fa9e6, entries=150, sequenceid=343, filesize=12.0 K 2024-11-12T00:54:11,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~399.33 KB/408912, currentSize=40.25 KB/41220 for eacf29d404b7a11371b9ac1286c14c2b in 1243ms, sequenceid=343, compaction requested=true 2024-11-12T00:54:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:54:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:11,102 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:54:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:11,102 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:54:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:11,103 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:11,104 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/A is initiating minor compaction (all files) 2024-11-12T00:54:11,104 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/A in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:11,104 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/c9531cd40bb84f73879a9fee4348c03a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/53ab977fac294beea903c8ac4425dacf, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/219873cc59574275a1e14a4ed4962a5f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=36.7 K 2024-11-12T00:54:11,104 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:11,104 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/B is initiating minor compaction (all files) 2024-11-12T00:54:11,104 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/B in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:11,104 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/225e4392ee1742aba8726ea9c4f14b62, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/f571c88616ab42a28ee760bcc6d31343, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/5668cd73cf33400691569016686b473a] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=36.7 K 2024-11-12T00:54:11,104 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9531cd40bb84f73879a9fee4348c03a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731372848724 2024-11-12T00:54:11,104 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 225e4392ee1742aba8726ea9c4f14b62, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731372848724 2024-11-12T00:54:11,104 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53ab977fac294beea903c8ac4425dacf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731372848838 2024-11-12T00:54:11,105 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting f571c88616ab42a28ee760bcc6d31343, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731372848838 2024-11-12T00:54:11,105 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 219873cc59574275a1e14a4ed4962a5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731372849524 2024-11-12T00:54:11,105 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 5668cd73cf33400691569016686b473a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731372849524 2024-11-12T00:54:11,127 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#A#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:11,128 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/310ea9f0d52d461b93a8e7ee63d962cd is 50, key is test_row_0/A:col10/1731372849524/Put/seqid=0 2024-11-12T00:54:11,130 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#B#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:11,131 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/9150547d60e7476ea8ccd7bb886e284d is 50, key is test_row_0/B:col10/1731372849524/Put/seqid=0 2024-11-12T00:54:11,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742134_1310 (size=13051) 2024-11-12T00:54:11,141 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/310ea9f0d52d461b93a8e7ee63d962cd as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/310ea9f0d52d461b93a8e7ee63d962cd 2024-11-12T00:54:11,145 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/A of eacf29d404b7a11371b9ac1286c14c2b into 310ea9f0d52d461b93a8e7ee63d962cd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:11,145 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:11,146 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/A, priority=13, startTime=1731372851102; duration=0sec 2024-11-12T00:54:11,146 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:11,146 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:A 2024-11-12T00:54:11,146 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:11,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742135_1311 (size=13051) 2024-11-12T00:54:11,147 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:11,147 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/C is initiating minor compaction (all files) 2024-11-12T00:54:11,148 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/C in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:11,148 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/c303672a26ac43ee820eaef305bdfda5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3133167ee5044785896419e442eb82b8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/73fc10e0ba304f4e9b0b140d022fa9e6] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=36.7 K 2024-11-12T00:54:11,148 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting c303672a26ac43ee820eaef305bdfda5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731372848724 2024-11-12T00:54:11,148 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3133167ee5044785896419e442eb82b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731372848838 2024-11-12T00:54:11,149 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73fc10e0ba304f4e9b0b140d022fa9e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731372849524 2024-11-12T00:54:11,151 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/9150547d60e7476ea8ccd7bb886e284d as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/9150547d60e7476ea8ccd7bb886e284d 2024-11-12T00:54:11,157 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#C#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:11,157 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/4a2ac573e5b9475588b0e46b94f288b9 is 50, key is test_row_0/C:col10/1731372849524/Put/seqid=0 2024-11-12T00:54:11,157 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/B of eacf29d404b7a11371b9ac1286c14c2b into 9150547d60e7476ea8ccd7bb886e284d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:11,157 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:11,157 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/B, priority=13, startTime=1731372851102; duration=0sec 2024-11-12T00:54:11,158 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:11,158 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:B 2024-11-12T00:54:11,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742136_1312 (size=13051) 2024-11-12T00:54:11,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T00:54:11,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:11,171 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-12T00:54:11,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:11,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:11,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:11,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:11,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:11,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:11,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/050a2a3925f44ef28243cf168a84bec0 is 50, key is test_row_0/A:col10/1731372849869/Put/seqid=0 2024-11-12T00:54:11,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742137_1313 (size=12301) 2024-11-12T00:54:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T00:54:11,568 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/4a2ac573e5b9475588b0e46b94f288b9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4a2ac573e5b9475588b0e46b94f288b9 2024-11-12T00:54:11,576 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/C of eacf29d404b7a11371b9ac1286c14c2b into 4a2ac573e5b9475588b0e46b94f288b9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:11,576 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:11,576 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/C, priority=13, startTime=1731372851102; duration=0sec 2024-11-12T00:54:11,577 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:11,577 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:C 2024-11-12T00:54:11,583 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/050a2a3925f44ef28243cf168a84bec0 2024-11-12T00:54:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/a6439057b5834efb83a33d6e783544b6 is 50, key is test_row_0/B:col10/1731372849869/Put/seqid=0 2024-11-12T00:54:11,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742138_1314 (size=12301) 2024-11-12T00:54:11,609 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/a6439057b5834efb83a33d6e783544b6 2024-11-12T00:54:11,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/81ca7e9187df49c0a0c56803c30ba276 is 50, key is test_row_0/C:col10/1731372849869/Put/seqid=0 2024-11-12T00:54:11,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742139_1315 (size=12301) 2024-11-12T00:54:11,626 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/81ca7e9187df49c0a0c56803c30ba276 2024-11-12T00:54:11,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/050a2a3925f44ef28243cf168a84bec0 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/050a2a3925f44ef28243cf168a84bec0 2024-11-12T00:54:11,641 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/050a2a3925f44ef28243cf168a84bec0, entries=150, sequenceid=354, filesize=12.0 K 2024-11-12T00:54:11,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/a6439057b5834efb83a33d6e783544b6 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a6439057b5834efb83a33d6e783544b6 2024-11-12T00:54:11,648 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a6439057b5834efb83a33d6e783544b6, entries=150, sequenceid=354, filesize=12.0 K 2024-11-12T00:54:11,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/81ca7e9187df49c0a0c56803c30ba276 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/81ca7e9187df49c0a0c56803c30ba276 2024-11-12T00:54:11,654 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/81ca7e9187df49c0a0c56803c30ba276, entries=150, sequenceid=354, filesize=12.0 K 2024-11-12T00:54:11,654 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for eacf29d404b7a11371b9ac1286c14c2b in 483ms, sequenceid=354, compaction requested=false 2024-11-12T00:54:11,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:11,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:11,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-12T00:54:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-12T00:54:11,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-12T00:54:11,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0980 sec 2024-11-12T00:54:11,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.1030 sec 2024-11-12T00:54:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:11,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T00:54:11,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:11,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:11,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:11,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:11,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:11,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:11,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/baf4edf22e3f445cb2cbd9fd2455a53e is 50, key is test_row_0/A:col10/1731372851683/Put/seqid=0 2024-11-12T00:54:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T00:54:11,697 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:11,697 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:54:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-12T00:54:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T00:54:11,702 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:54:11,703 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:54:11,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:54:11,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742140_1316 (size=12301) 2024-11-12T00:54:11,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/baf4edf22e3f445cb2cbd9fd2455a53e 2024-11-12T00:54:11,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/af462253ac6e4b2d9d935dba3ca9b804 is 50, key is test_row_0/B:col10/1731372851683/Put/seqid=0 2024-11-12T00:54:11,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742141_1317 (size=12301) 2024-11-12T00:54:11,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/af462253ac6e4b2d9d935dba3ca9b804 2024-11-12T00:54:11,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/297b8a76802949ffb50b30cf482e491b is 50, key is test_row_0/C:col10/1731372851683/Put/seqid=0 2024-11-12T00:54:11,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742142_1318 (size=12301) 2024-11-12T00:54:11,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/297b8a76802949ffb50b30cf482e491b 2024-11-12T00:54:11,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/baf4edf22e3f445cb2cbd9fd2455a53e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/baf4edf22e3f445cb2cbd9fd2455a53e 2024-11-12T00:54:11,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/baf4edf22e3f445cb2cbd9fd2455a53e, entries=150, sequenceid=365, filesize=12.0 K 2024-11-12T00:54:11,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/af462253ac6e4b2d9d935dba3ca9b804 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/af462253ac6e4b2d9d935dba3ca9b804 2024-11-12T00:54:11,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/af462253ac6e4b2d9d935dba3ca9b804, entries=150, sequenceid=365, filesize=12.0 K 2024-11-12T00:54:11,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/297b8a76802949ffb50b30cf482e491b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/297b8a76802949ffb50b30cf482e491b 2024-11-12T00:54:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T00:54:11,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/297b8a76802949ffb50b30cf482e491b, entries=150, sequenceid=365, filesize=12.0 K 2024-11-12T00:54:11,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=93.93 KB/96180 for eacf29d404b7a11371b9ac1286c14c2b in 120ms, sequenceid=365, compaction requested=true 2024-11-12T00:54:11,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:11,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:54:11,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:11,809 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:11,809 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:11,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:54:11,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:11,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:54:11,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:11,812 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:11,812 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/A is initiating minor compaction (all files) 2024-11-12T00:54:11,812 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:11,812 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/B is initiating minor compaction (all files) 2024-11-12T00:54:11,812 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/A in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:11,812 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/B in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:11,812 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/9150547d60e7476ea8ccd7bb886e284d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a6439057b5834efb83a33d6e783544b6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/af462253ac6e4b2d9d935dba3ca9b804] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=36.8 K 2024-11-12T00:54:11,812 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/310ea9f0d52d461b93a8e7ee63d962cd, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/050a2a3925f44ef28243cf168a84bec0, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/baf4edf22e3f445cb2cbd9fd2455a53e] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=36.8 K 2024-11-12T00:54:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:11,812 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 9150547d60e7476ea8ccd7bb886e284d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731372849524 2024-11-12T00:54:11,812 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 310ea9f0d52d461b93a8e7ee63d962cd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731372849524 2024-11-12T00:54:11,813 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting a6439057b5834efb83a33d6e783544b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731372849868 2024-11-12T00:54:11,813 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 050a2a3925f44ef28243cf168a84bec0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731372849868 2024-11-12T00:54:11,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T00:54:11,813 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting baf4edf22e3f445cb2cbd9fd2455a53e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731372851669 2024-11-12T00:54:11,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:11,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:11,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:11,813 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting af462253ac6e4b2d9d935dba3ca9b804, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731372851669 2024-11-12T00:54:11,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:11,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:11,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:11,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/0bbb82f6255846c2a6d5c24163f0fc88 is 50, key is test_row_0/A:col10/1731372851810/Put/seqid=0 2024-11-12T00:54:11,835 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#A#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:11,836 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/8343dd93d5cf42d5b3c84d7501c3d409 is 50, key is test_row_0/A:col10/1731372851683/Put/seqid=0 2024-11-12T00:54:11,855 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#B#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:11,856 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/93f5d6fd5cb84f88bc98df9f96afe407 is 50, key is test_row_0/B:col10/1731372851683/Put/seqid=0 2024-11-12T00:54:11,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T00:54:11,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:11,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:11,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:11,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:11,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:11,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742143_1319 (size=12301) 2024-11-12T00:54:11,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/0bbb82f6255846c2a6d5c24163f0fc88 2024-11-12T00:54:11,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742144_1320 (size=13153) 2024-11-12T00:54:11,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/09f371b985284dd19af3d5a622f1d0ea is 50, key is test_row_0/B:col10/1731372851810/Put/seqid=0 2024-11-12T00:54:11,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372911921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:11,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742145_1321 (size=13153) 2024-11-12T00:54:11,942 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/93f5d6fd5cb84f88bc98df9f96afe407 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/93f5d6fd5cb84f88bc98df9f96afe407 2024-11-12T00:54:11,949 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/B of eacf29d404b7a11371b9ac1286c14c2b into 93f5d6fd5cb84f88bc98df9f96afe407(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:11,949 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:11,949 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/B, priority=13, startTime=1731372851809; duration=0sec 2024-11-12T00:54:11,949 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:11,949 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:B 2024-11-12T00:54:11,950 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:11,951 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:11,951 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/C is initiating minor compaction (all files) 2024-11-12T00:54:11,951 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/C in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:11,951 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4a2ac573e5b9475588b0e46b94f288b9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/81ca7e9187df49c0a0c56803c30ba276, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/297b8a76802949ffb50b30cf482e491b] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=36.8 K 2024-11-12T00:54:11,951 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a2ac573e5b9475588b0e46b94f288b9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731372849524 2024-11-12T00:54:11,952 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 81ca7e9187df49c0a0c56803c30ba276, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731372849868 2024-11-12T00:54:11,952 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 297b8a76802949ffb50b30cf482e491b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731372851669 2024-11-12T00:54:11,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742146_1322 (size=12301) 2024-11-12T00:54:11,967 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#C#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:11,968 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/6930679620e64f25bd959f3918ba4c21 is 50, key is test_row_0/C:col10/1731372851683/Put/seqid=0 2024-11-12T00:54:11,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742147_1323 (size=13153) 2024-11-12T00:54:11,986 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/6930679620e64f25bd959f3918ba4c21 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6930679620e64f25bd959f3918ba4c21 2024-11-12T00:54:11,992 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/C of eacf29d404b7a11371b9ac1286c14c2b into 6930679620e64f25bd959f3918ba4c21(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:11,992 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:11,992 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/C, priority=13, startTime=1731372851809; duration=0sec 2024-11-12T00:54:11,993 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:11,993 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:C 2024-11-12T00:54:12,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T00:54:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:12,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:12,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:12,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:12,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T00:54:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372912025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:12,028 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in eacf29d404b7a11371b9ac1286c14c2b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:12,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372912036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:12,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372912036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,038 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_1' in eacf29d404b7a11371b9ac1286c14c2b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372912036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:12,039 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_1' in eacf29d404b7a11371b9ac1286c14c2b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:12,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T00:54:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:12,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:12,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372912245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:12,306 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/8343dd93d5cf42d5b3c84d7501c3d409 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8343dd93d5cf42d5b3c84d7501c3d409 2024-11-12T00:54:12,311 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/A of eacf29d404b7a11371b9ac1286c14c2b into 8343dd93d5cf42d5b3c84d7501c3d409(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:12,311 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:12,311 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/A, priority=13, startTime=1731372851809; duration=0sec 2024-11-12T00:54:12,311 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:12,311 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:A 2024-11-12T00:54:12,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T00:54:12,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:12,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:12,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:12,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:12,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T00:54:12,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/09f371b985284dd19af3d5a622f1d0ea 2024-11-12T00:54:12,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/947200d29a334d3daf88ff2913f0a27f is 50, key is test_row_0/C:col10/1731372851810/Put/seqid=0 2024-11-12T00:54:12,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742148_1324 (size=12301) 2024-11-12T00:54:12,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/947200d29a334d3daf88ff2913f0a27f 2024-11-12T00:54:12,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/0bbb82f6255846c2a6d5c24163f0fc88 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/0bbb82f6255846c2a6d5c24163f0fc88 2024-11-12T00:54:12,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/0bbb82f6255846c2a6d5c24163f0fc88, entries=150, sequenceid=383, filesize=12.0 K 2024-11-12T00:54:12,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/09f371b985284dd19af3d5a622f1d0ea as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/09f371b985284dd19af3d5a622f1d0ea 2024-11-12T00:54:12,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/09f371b985284dd19af3d5a622f1d0ea, entries=150, sequenceid=383, filesize=12.0 K 2024-11-12T00:54:12,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/947200d29a334d3daf88ff2913f0a27f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/947200d29a334d3daf88ff2913f0a27f 2024-11-12T00:54:12,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/947200d29a334d3daf88ff2913f0a27f, entries=150, sequenceid=383, filesize=12.0 K 2024-11-12T00:54:12,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for eacf29d404b7a11371b9ac1286c14c2b in 586ms, sequenceid=383, compaction requested=false 2024-11-12T00:54:12,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:12,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T00:54:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:12,472 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T00:54:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:12,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/8957c8e334564884b6a86e518a407cda is 50, key is test_row_0/A:col10/1731372851883/Put/seqid=0 2024-11-12T00:54:12,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742149_1325 (size=12301) 2024-11-12T00:54:12,483 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/8957c8e334564884b6a86e518a407cda 2024-11-12T00:54:12,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/6e67bd749dd64c4bb2f8043f7d114815 is 50, key is test_row_0/B:col10/1731372851883/Put/seqid=0 2024-11-12T00:54:12,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742150_1326 (size=12301) 2024-11-12T00:54:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:12,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:12,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372912621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:12,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372912735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T00:54:12,895 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/6e67bd749dd64c4bb2f8043f7d114815 2024-11-12T00:54:12,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/5f29ad508f0548c2870ac1c337e1e0b5 is 50, key is test_row_0/C:col10/1731372851883/Put/seqid=0 2024-11-12T00:54:12,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742151_1327 (size=12301) 2024-11-12T00:54:12,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:12,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372912946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:12,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:12,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:13,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372913255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:13,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:13,311 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/5f29ad508f0548c2870ac1c337e1e0b5 2024-11-12T00:54:13,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/8957c8e334564884b6a86e518a407cda as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8957c8e334564884b6a86e518a407cda 2024-11-12T00:54:13,332 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8957c8e334564884b6a86e518a407cda, entries=150, sequenceid=400, filesize=12.0 K 2024-11-12T00:54:13,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/6e67bd749dd64c4bb2f8043f7d114815 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/6e67bd749dd64c4bb2f8043f7d114815 2024-11-12T00:54:13,338 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/6e67bd749dd64c4bb2f8043f7d114815, entries=150, sequenceid=400, filesize=12.0 K 2024-11-12T00:54:13,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/5f29ad508f0548c2870ac1c337e1e0b5 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/5f29ad508f0548c2870ac1c337e1e0b5 2024-11-12T00:54:13,344 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/5f29ad508f0548c2870ac1c337e1e0b5, entries=150, sequenceid=400, filesize=12.0 K 2024-11-12T00:54:13,348 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for eacf29d404b7a11371b9ac1286c14c2b in 876ms, sequenceid=400, compaction requested=true 2024-11-12T00:54:13,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:13,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:13,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-12T00:54:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-12T00:54:13,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-12T00:54:13,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6470 sec 2024-11-12T00:54:13,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.6560 sec 2024-11-12T00:54:13,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T00:54:13,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:13,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:13,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:13,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:13,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:13,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:13,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/91a9fbfe55d243428ae521918176ee06 is 50, key is test_row_0/A:col10/1731372852615/Put/seqid=0 2024-11-12T00:54:13,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742152_1328 (size=14741) 2024-11-12T00:54:13,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:13,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372913830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:13,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:13,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:13,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T00:54:13,856 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:13,857 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:54:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-12T00:54:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T00:54:13,860 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:54:13,862 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:54:13,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:54:13,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372913935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:13,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:13,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:13,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T00:54:14,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T00:54:14,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:14,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:14,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:14,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:14,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:14,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:14,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39628 deadline: 1731372914046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:14,049 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in eacf29d404b7a11371b9ac1286c14c2b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:14,054 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T00:54:14,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39698 deadline: 1731372914056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:14,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1731372914056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:14,061 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_1' in eacf29d404b7a11371b9ac1286c14c2b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:14,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:14,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:14,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:14,061 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_1' in eacf29d404b7a11371b9ac1286c14c2b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:14,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:14,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:14,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:14,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372914146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:14,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:14,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T00:54:14,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:14,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:14,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:14,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:14,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T00:54:14,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/91a9fbfe55d243428ae521918176ee06 2024-11-12T00:54:14,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/19efaf7dddbe481eb3562e45d5d1b586 is 50, key is test_row_0/B:col10/1731372852615/Put/seqid=0 2024-11-12T00:54:14,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742153_1329 (size=12301) 2024-11-12T00:54:14,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T00:54:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:14,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:14,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:14,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372914465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:14,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T00:54:14,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:14,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:14,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:14,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:14,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T00:54:14,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T00:54:14,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:14,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:14,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:14,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:14,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:14,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/19efaf7dddbe481eb3562e45d5d1b586 2024-11-12T00:54:14,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/7b6589a9b9bc4062848484201798494b is 50, key is test_row_0/C:col10/1731372852615/Put/seqid=0 2024-11-12T00:54:14,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742154_1330 (size=12301) 2024-11-12T00:54:14,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/7b6589a9b9bc4062848484201798494b 2024-11-12T00:54:14,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/91a9fbfe55d243428ae521918176ee06 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/91a9fbfe55d243428ae521918176ee06 2024-11-12T00:54:14,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/91a9fbfe55d243428ae521918176ee06, entries=200, sequenceid=419, filesize=14.4 K 2024-11-12T00:54:14,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/19efaf7dddbe481eb3562e45d5d1b586 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/19efaf7dddbe481eb3562e45d5d1b586 2024-11-12T00:54:14,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/19efaf7dddbe481eb3562e45d5d1b586, entries=150, sequenceid=419, filesize=12.0 K 2024-11-12T00:54:14,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/7b6589a9b9bc4062848484201798494b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b6589a9b9bc4062848484201798494b 2024-11-12T00:54:14,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b6589a9b9bc4062848484201798494b, entries=150, sequenceid=419, filesize=12.0 K 2024-11-12T00:54:14,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for eacf29d404b7a11371b9ac1286c14c2b in 1004ms, sequenceid=419, compaction requested=true 2024-11-12T00:54:14,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:14,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:54:14,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:14,772 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:54:14,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:54:14,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:14,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:54:14,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T00:54:14,773 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:54:14,774 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52496 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:54:14,774 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/A is initiating minor compaction (all files) 2024-11-12T00:54:14,774 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/A in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:14,774 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8343dd93d5cf42d5b3c84d7501c3d409, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/0bbb82f6255846c2a6d5c24163f0fc88, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8957c8e334564884b6a86e518a407cda, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/91a9fbfe55d243428ae521918176ee06] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=51.3 K 2024-11-12T00:54:14,775 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:54:14,775 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 8343dd93d5cf42d5b3c84d7501c3d409, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731372851669 2024-11-12T00:54:14,775 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/B is initiating minor compaction (all files) 2024-11-12T00:54:14,775 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/B in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:14,775 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/93f5d6fd5cb84f88bc98df9f96afe407, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/09f371b985284dd19af3d5a622f1d0ea, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/6e67bd749dd64c4bb2f8043f7d114815, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/19efaf7dddbe481eb3562e45d5d1b586] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=48.9 K 2024-11-12T00:54:14,777 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bbb82f6255846c2a6d5c24163f0fc88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731372851720 2024-11-12T00:54:14,777 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93f5d6fd5cb84f88bc98df9f96afe407, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731372851669 2024-11-12T00:54:14,777 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09f371b985284dd19af3d5a622f1d0ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731372851720 2024-11-12T00:54:14,777 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 8957c8e334564884b6a86e518a407cda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731372851883 2024-11-12T00:54:14,777 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e67bd749dd64c4bb2f8043f7d114815, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731372851883 2024-11-12T00:54:14,777 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 91a9fbfe55d243428ae521918176ee06, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731372852596 2024-11-12T00:54:14,778 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19efaf7dddbe481eb3562e45d5d1b586, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731372852615 2024-11-12T00:54:14,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T00:54:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:14,785 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T00:54:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:14,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:14,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/bc9a7ae1e05046f1bc42a262df232c65 is 50, key is test_row_0/A:col10/1731372853828/Put/seqid=0 2024-11-12T00:54:14,811 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#B#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:14,812 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/1a428da803fc4549b73a1af6f094b81e is 50, key is test_row_0/B:col10/1731372852615/Put/seqid=0 2024-11-12T00:54:14,848 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#A#compaction#287 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:14,848 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/14c3e9d9e43c4f9fae79083463d92962 is 50, key is test_row_0/A:col10/1731372852615/Put/seqid=0 2024-11-12T00:54:14,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742155_1331 (size=12301) 2024-11-12T00:54:14,877 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/bc9a7ae1e05046f1bc42a262df232c65 2024-11-12T00:54:14,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742156_1332 (size=13289) 2024-11-12T00:54:14,897 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/1a428da803fc4549b73a1af6f094b81e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/1a428da803fc4549b73a1af6f094b81e 2024-11-12T00:54:14,902 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/B of eacf29d404b7a11371b9ac1286c14c2b into 1a428da803fc4549b73a1af6f094b81e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:14,903 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:14,903 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/B, priority=12, startTime=1731372854772; duration=0sec 2024-11-12T00:54:14,903 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:14,903 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:B 2024-11-12T00:54:14,903 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:54:14,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/4760b0bad68e4c6390d63dbd06c9bd96 is 50, key is test_row_0/B:col10/1731372853828/Put/seqid=0 2024-11-12T00:54:14,905 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:54:14,905 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/C is initiating minor compaction (all files) 2024-11-12T00:54:14,905 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/C in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:14,905 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6930679620e64f25bd959f3918ba4c21, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/947200d29a334d3daf88ff2913f0a27f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/5f29ad508f0548c2870ac1c337e1e0b5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b6589a9b9bc4062848484201798494b] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=48.9 K 2024-11-12T00:54:14,907 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6930679620e64f25bd959f3918ba4c21, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731372851669 2024-11-12T00:54:14,907 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 947200d29a334d3daf88ff2913f0a27f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731372851720 2024-11-12T00:54:14,908 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f29ad508f0548c2870ac1c337e1e0b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731372851883 2024-11-12T00:54:14,909 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b6589a9b9bc4062848484201798494b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731372852615 2024-11-12T00:54:14,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742157_1333 (size=13289) 2024-11-12T00:54:14,936 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#C#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:14,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742158_1334 (size=12301) 2024-11-12T00:54:14,937 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/cbd5ea5bf0df4fe3a7ea599533853f75 is 50, key is test_row_0/C:col10/1731372852615/Put/seqid=0 2024-11-12T00:54:14,938 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/4760b0bad68e4c6390d63dbd06c9bd96 2024-11-12T00:54:14,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742159_1335 (size=13289) 2024-11-12T00:54:14,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/2116f167bba7446099849a6b30d62ae5 is 50, key is test_row_0/C:col10/1731372853828/Put/seqid=0 2024-11-12T00:54:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:14,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. as already flushing 2024-11-12T00:54:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T00:54:15,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742160_1336 (size=12301) 2024-11-12T00:54:15,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372915087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:15,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372915196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:15,328 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/14c3e9d9e43c4f9fae79083463d92962 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/14c3e9d9e43c4f9fae79083463d92962 2024-11-12T00:54:15,333 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/A of eacf29d404b7a11371b9ac1286c14c2b into 14c3e9d9e43c4f9fae79083463d92962(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:15,333 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:15,333 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/A, priority=12, startTime=1731372854772; duration=0sec 2024-11-12T00:54:15,333 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:15,333 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:A 2024-11-12T00:54:15,372 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/cbd5ea5bf0df4fe3a7ea599533853f75 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/cbd5ea5bf0df4fe3a7ea599533853f75 2024-11-12T00:54:15,378 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/C of eacf29d404b7a11371b9ac1286c14c2b into cbd5ea5bf0df4fe3a7ea599533853f75(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:15,378 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:15,378 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/C, priority=12, startTime=1731372854772; duration=0sec 2024-11-12T00:54:15,378 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:15,378 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:C 2024-11-12T00:54:15,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:15,409 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/2116f167bba7446099849a6b30d62ae5 2024-11-12T00:54:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372915405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:15,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/bc9a7ae1e05046f1bc42a262df232c65 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/bc9a7ae1e05046f1bc42a262df232c65 2024-11-12T00:54:15,425 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/bc9a7ae1e05046f1bc42a262df232c65, entries=150, sequenceid=432, filesize=12.0 K 2024-11-12T00:54:15,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/4760b0bad68e4c6390d63dbd06c9bd96 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/4760b0bad68e4c6390d63dbd06c9bd96 2024-11-12T00:54:15,431 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/4760b0bad68e4c6390d63dbd06c9bd96, entries=150, sequenceid=432, filesize=12.0 K 2024-11-12T00:54:15,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/2116f167bba7446099849a6b30d62ae5 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/2116f167bba7446099849a6b30d62ae5 2024-11-12T00:54:15,438 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/2116f167bba7446099849a6b30d62ae5, entries=150, sequenceid=432, filesize=12.0 K 2024-11-12T00:54:15,440 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for eacf29d404b7a11371b9ac1286c14c2b in 655ms, sequenceid=432, compaction requested=false 2024-11-12T00:54:15,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:15,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:15,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-12T00:54:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-12T00:54:15,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-12T00:54:15,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5800 sec 2024-11-12T00:54:15,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.5890 sec 2024-11-12T00:54:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:15,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T00:54:15,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:15,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:15,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:15,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:15,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:15,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:15,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/5c5893b493a649c0be1fcc363da30de5 is 50, key is test_row_0/A:col10/1731372855717/Put/seqid=0 2024-11-12T00:54:15,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742161_1337 (size=12301) 2024-11-12T00:54:15,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/5c5893b493a649c0be1fcc363da30de5 2024-11-12T00:54:15,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/613e110da615468690595c5adfb92a83 is 50, key is test_row_0/B:col10/1731372855717/Put/seqid=0 2024-11-12T00:54:15,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:15,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372915770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:15,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742162_1338 (size=12301) 2024-11-12T00:54:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:15,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372915896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:16,014 INFO [Thread-1146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1146. 2024-11-12T00:54:16,014 DEBUG [Thread-1146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:16,014 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:16,014 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:16,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T00:54:16,016 INFO [Thread-1144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1144. 2024-11-12T00:54:16,016 DEBUG [Thread-1144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:16,016 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:16,016 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:16,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:16,016 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:16,018 INFO [Thread-1150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1150. 2024-11-12T00:54:16,018 DEBUG [Thread-1150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:16,018 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:16,018 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:16,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:16,018 INFO [Thread-1142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1142. 2024-11-12T00:54:16,018 DEBUG [Thread-1142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:16,018 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:16,018 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:16,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:16,020 INFO [Thread-1148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1148. 2024-11-12T00:54:16,020 DEBUG [Thread-1148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:16,020 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:16,020 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:16,020 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:16,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:16,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372916106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:16,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/613e110da615468690595c5adfb92a83 2024-11-12T00:54:16,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/1a2682626abb4789b77c699ea7f5efe0 is 50, key is test_row_0/C:col10/1731372855717/Put/seqid=0 2024-11-12T00:54:16,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742163_1339 (size=12301) 2024-11-12T00:54:16,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1731372916416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:16,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:16,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/1a2682626abb4789b77c699ea7f5efe0 2024-11-12T00:54:16,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/5c5893b493a649c0be1fcc363da30de5 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/5c5893b493a649c0be1fcc363da30de5 2024-11-12T00:54:16,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/5c5893b493a649c0be1fcc363da30de5, entries=150, sequenceid=455, filesize=12.0 K 2024-11-12T00:54:16,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/613e110da615468690595c5adfb92a83 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/613e110da615468690595c5adfb92a83 2024-11-12T00:54:16,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/613e110da615468690595c5adfb92a83, entries=150, sequenceid=455, filesize=12.0 K 2024-11-12T00:54:16,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/1a2682626abb4789b77c699ea7f5efe0 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/1a2682626abb4789b77c699ea7f5efe0 2024-11-12T00:54:16,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/1a2682626abb4789b77c699ea7f5efe0, entries=150, sequenceid=455, filesize=12.0 K 2024-11-12T00:54:16,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for eacf29d404b7a11371b9ac1286c14c2b in 896ms, sequenceid=455, compaction requested=true 2024-11-12T00:54:16,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:16,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:54:16,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:16,614 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:16,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:54:16,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:16,614 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:16,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store eacf29d404b7a11371b9ac1286c14c2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:54:16,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:16,615 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:16,615 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:16,616 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/B is initiating minor compaction (all files) 2024-11-12T00:54:16,616 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/A is initiating minor compaction (all files) 2024-11-12T00:54:16,616 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/A in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:16,616 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/B in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:16,616 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/14c3e9d9e43c4f9fae79083463d92962, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/bc9a7ae1e05046f1bc42a262df232c65, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/5c5893b493a649c0be1fcc363da30de5] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=37.0 K 2024-11-12T00:54:16,616 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/1a428da803fc4549b73a1af6f094b81e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/4760b0bad68e4c6390d63dbd06c9bd96, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/613e110da615468690595c5adfb92a83] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=37.0 K 2024-11-12T00:54:16,616 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a428da803fc4549b73a1af6f094b81e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731372852615 2024-11-12T00:54:16,616 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14c3e9d9e43c4f9fae79083463d92962, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731372852615 2024-11-12T00:54:16,616 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc9a7ae1e05046f1bc42a262df232c65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731372853817 2024-11-12T00:54:16,616 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 4760b0bad68e4c6390d63dbd06c9bd96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731372853817 2024-11-12T00:54:16,616 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c5893b493a649c0be1fcc363da30de5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731372855074 2024-11-12T00:54:16,617 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 613e110da615468690595c5adfb92a83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731372855074 2024-11-12T00:54:16,634 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#A#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:16,634 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:16,635 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/771c60f853524ad7a31439e1e02ffae0 is 50, key is test_row_0/B:col10/1731372855717/Put/seqid=0 2024-11-12T00:54:16,635 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/6775c14c2a004c70857918b599c8e6cc is 50, key is test_row_0/A:col10/1731372855717/Put/seqid=0 2024-11-12T00:54:16,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742165_1341 (size=13391) 2024-11-12T00:54:16,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742164_1340 (size=13391) 2024-11-12T00:54:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:16,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T00:54:16,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:16,928 INFO [Thread-1135 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1135. 2024-11-12T00:54:16,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:16,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:16,928 DEBUG [Thread-1135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:16,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:16,928 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:16,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:16,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:16,928 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:16,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:16,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/ac443a52c5e34578b35ce8d829165dc7 is 50, key is test_row_0/A:col10/1731372855747/Put/seqid=0 2024-11-12T00:54:16,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742166_1342 (size=12301) 2024-11-12T00:54:17,052 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/6775c14c2a004c70857918b599c8e6cc as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/6775c14c2a004c70857918b599c8e6cc 2024-11-12T00:54:17,053 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/771c60f853524ad7a31439e1e02ffae0 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/771c60f853524ad7a31439e1e02ffae0 2024-11-12T00:54:17,058 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/A of eacf29d404b7a11371b9ac1286c14c2b into 6775c14c2a004c70857918b599c8e6cc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:17,058 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:17,058 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/A, priority=13, startTime=1731372856614; duration=0sec 2024-11-12T00:54:17,059 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:17,059 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:A 2024-11-12T00:54:17,059 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:17,061 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/B of eacf29d404b7a11371b9ac1286c14c2b into 771c60f853524ad7a31439e1e02ffae0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:17,061 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:17,061 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/B, priority=13, startTime=1731372856614; duration=0sec 2024-11-12T00:54:17,061 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:17,061 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:17,061 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:B 2024-11-12T00:54:17,061 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): eacf29d404b7a11371b9ac1286c14c2b/C is initiating minor compaction (all files) 2024-11-12T00:54:17,061 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of eacf29d404b7a11371b9ac1286c14c2b/C in TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:17,061 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/cbd5ea5bf0df4fe3a7ea599533853f75, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/2116f167bba7446099849a6b30d62ae5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/1a2682626abb4789b77c699ea7f5efe0] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp, totalSize=37.0 K 2024-11-12T00:54:17,061 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbd5ea5bf0df4fe3a7ea599533853f75, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731372852615 2024-11-12T00:54:17,062 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2116f167bba7446099849a6b30d62ae5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731372853817 2024-11-12T00:54:17,062 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a2682626abb4789b77c699ea7f5efe0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731372855074 2024-11-12T00:54:17,075 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): eacf29d404b7a11371b9ac1286c14c2b#C#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:17,075 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/cf31e1c10e7546ad9a1cc08e91b72e5a is 50, key is test_row_0/C:col10/1731372855717/Put/seqid=0 2024-11-12T00:54:17,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742167_1343 (size=13391) 2024-11-12T00:54:17,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/ac443a52c5e34578b35ce8d829165dc7 2024-11-12T00:54:17,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/a549a83fcf4b49d999c6659cd00e4a4d is 50, key is test_row_0/B:col10/1731372855747/Put/seqid=0 2024-11-12T00:54:17,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742168_1344 (size=12301) 2024-11-12T00:54:17,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/a549a83fcf4b49d999c6659cd00e4a4d 2024-11-12T00:54:17,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/0c9434a8dd204b6ebb21db2a7da24244 is 50, key is test_row_0/C:col10/1731372855747/Put/seqid=0 2024-11-12T00:54:17,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742169_1345 (size=12301) 2024-11-12T00:54:17,488 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/cf31e1c10e7546ad9a1cc08e91b72e5a as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/cf31e1c10e7546ad9a1cc08e91b72e5a 2024-11-12T00:54:17,494 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eacf29d404b7a11371b9ac1286c14c2b/C of eacf29d404b7a11371b9ac1286c14c2b into cf31e1c10e7546ad9a1cc08e91b72e5a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:17,494 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:17,494 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., storeName=eacf29d404b7a11371b9ac1286c14c2b/C, priority=13, startTime=1731372856614; duration=0sec 2024-11-12T00:54:17,494 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:17,494 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: eacf29d404b7a11371b9ac1286c14c2b:C 2024-11-12T00:54:17,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/0c9434a8dd204b6ebb21db2a7da24244 2024-11-12T00:54:17,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/ac443a52c5e34578b35ce8d829165dc7 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/ac443a52c5e34578b35ce8d829165dc7 2024-11-12T00:54:17,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/ac443a52c5e34578b35ce8d829165dc7, entries=150, sequenceid=468, filesize=12.0 K 2024-11-12T00:54:17,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/a549a83fcf4b49d999c6659cd00e4a4d as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a549a83fcf4b49d999c6659cd00e4a4d 2024-11-12T00:54:17,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a549a83fcf4b49d999c6659cd00e4a4d, entries=150, sequenceid=468, filesize=12.0 K 2024-11-12T00:54:17,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/0c9434a8dd204b6ebb21db2a7da24244 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/0c9434a8dd204b6ebb21db2a7da24244 2024-11-12T00:54:17,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/0c9434a8dd204b6ebb21db2a7da24244, entries=150, sequenceid=468, filesize=12.0 K 2024-11-12T00:54:17,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for eacf29d404b7a11371b9ac1286c14c2b in 864ms, sequenceid=468, compaction requested=false 2024-11-12T00:54:17,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for eacf29d404b7a11371b9ac1286c14c2b: 2024-11-12T00:54:18,090 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-11-12T00:54:18,090 INFO [Thread-1133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1133. 2024-11-12T00:54:18,090 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:18,091 DEBUG [Thread-1133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:18,091 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:18,091 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:18,091 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:18,091 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:18,091 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:18,091 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:18,098 INFO [Thread-1139 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1139. 2024-11-12T00:54:18,098 DEBUG [Thread-1139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:18,098 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:18,098 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:18,098 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:20,591 INFO [Thread-1131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1131. 2024-11-12T00:54:20,591 DEBUG [Thread-1131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:20,591 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:20,591 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:20,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:20,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T00:54:20,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-12T00:54:20,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-12T00:54:20,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 159 2024-11-12T00:54:20,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-12T00:54:20,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-12T00:54:20,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T00:54:20,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3048 2024-11-12T00:54:20,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3058 2024-11-12T00:54:20,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3095 2024-11-12T00:54:20,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3052 2024-11-12T00:54:20,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3054 2024-11-12T00:54:20,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T00:54:20,593 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T00:54:20,593 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T00:54:20,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:20,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:20,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:20,595 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T00:54:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T00:54:20,599 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372860599"}]},"ts":"1731372860599"} 2024-11-12T00:54:20,601 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T00:54:20,601 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T00:54:20,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T00:54:20,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eacf29d404b7a11371b9ac1286c14c2b, UNASSIGN}] 2024-11-12T00:54:20,604 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eacf29d404b7a11371b9ac1286c14c2b, UNASSIGN 2024-11-12T00:54:20,605 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=eacf29d404b7a11371b9ac1286c14c2b, regionState=CLOSING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:20,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=eacf29d404b7a11371b9ac1286c14c2b, UNASSIGN because future has completed 2024-11-12T00:54:20,607 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T00:54:20,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:54:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T00:54:20,761 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:20,761 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T00:54:20,761 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing eacf29d404b7a11371b9ac1286c14c2b, disabling compactions & flushes 2024-11-12T00:54:20,761 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:20,761 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:20,761 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. after waiting 0 ms 2024-11-12T00:54:20,761 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:20,762 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(2902): Flushing eacf29d404b7a11371b9ac1286c14c2b 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T00:54:20,762 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=A 2024-11-12T00:54:20,762 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:20,762 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=B 2024-11-12T00:54:20,762 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:20,762 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK eacf29d404b7a11371b9ac1286c14c2b, store=C 2024-11-12T00:54:20,762 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:20,769 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/2752580f13e24f7084aaf7bf9c7228dc is 50, key is test_row_0/A:col10/1731372860587/Put/seqid=0 2024-11-12T00:54:20,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742170_1346 (size=12301) 2024-11-12T00:54:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T00:54:21,174 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/2752580f13e24f7084aaf7bf9c7228dc 2024-11-12T00:54:21,181 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/9b9fe71cdf974258b637e48691e7329a is 50, key is test_row_0/B:col10/1731372860587/Put/seqid=0 2024-11-12T00:54:21,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742171_1347 (size=12301) 2024-11-12T00:54:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T00:54:21,585 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/9b9fe71cdf974258b637e48691e7329a 2024-11-12T00:54:21,591 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/37d5e874a94f4f01b1e2397dfd1bfe20 is 50, key is test_row_0/C:col10/1731372860587/Put/seqid=0 2024-11-12T00:54:21,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742172_1348 (size=12301) 2024-11-12T00:54:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T00:54:21,996 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/37d5e874a94f4f01b1e2397dfd1bfe20 2024-11-12T00:54:22,007 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/A/2752580f13e24f7084aaf7bf9c7228dc as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/2752580f13e24f7084aaf7bf9c7228dc 2024-11-12T00:54:22,012 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/2752580f13e24f7084aaf7bf9c7228dc, entries=150, sequenceid=478, filesize=12.0 K 2024-11-12T00:54:22,013 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/B/9b9fe71cdf974258b637e48691e7329a as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/9b9fe71cdf974258b637e48691e7329a 2024-11-12T00:54:22,018 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/9b9fe71cdf974258b637e48691e7329a, entries=150, sequenceid=478, filesize=12.0 K 2024-11-12T00:54:22,019 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/.tmp/C/37d5e874a94f4f01b1e2397dfd1bfe20 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/37d5e874a94f4f01b1e2397dfd1bfe20 2024-11-12T00:54:22,023 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/37d5e874a94f4f01b1e2397dfd1bfe20, entries=150, sequenceid=478, filesize=12.0 K 2024-11-12T00:54:22,024 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for eacf29d404b7a11371b9ac1286c14c2b in 1263ms, sequenceid=478, compaction requested=true 2024-11-12T00:54:22,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/dc7b0df68777418d8a6efc80e8ea95df, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/a91eccadcbf44278bdbea0511ee3ae40, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/63c4a59f03374dddb1877bebf5624918, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b3ef2c7a0cf949e89a99a19556d7d9d2, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/6c8781923b444095bcd31f35adcf17e5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/81fc997e00354e3495c8646abf384a75, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b0b0b01cfebf42648aa349162b3602a9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7667e257b8ef428d95ff52464c164cb4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/5ff0ff789cbc433ab50d7158b8b4ead4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/6529a82fd4a845d5a08a1363e8e9e2c5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/a2c6c1198a6b44b1bdc64bff121df735, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b4221fa919994253a0b758781a445f94, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7b9fbc2a17b84bd59aa56da0b0d20835, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/cd4df67709d1402a9151b8156620f2b7, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8692685d9acf4d8e83a185f59144e706, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/049880fa40bf40779bf7450dc1e6c22d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7cff125bb0234640b13080906355b854, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/bb1928b77c3d4c8c93026549187d7b1e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/c9531cd40bb84f73879a9fee4348c03a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/be1de6f289794b95b7701e47f5e72fbe, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/53ab977fac294beea903c8ac4425dacf, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/310ea9f0d52d461b93a8e7ee63d962cd, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/219873cc59574275a1e14a4ed4962a5f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/050a2a3925f44ef28243cf168a84bec0, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8343dd93d5cf42d5b3c84d7501c3d409, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/baf4edf22e3f445cb2cbd9fd2455a53e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/0bbb82f6255846c2a6d5c24163f0fc88, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8957c8e334564884b6a86e518a407cda, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/91a9fbfe55d243428ae521918176ee06, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/14c3e9d9e43c4f9fae79083463d92962, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/bc9a7ae1e05046f1bc42a262df232c65, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/5c5893b493a649c0be1fcc363da30de5] to archive 2024-11-12T00:54:22,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:54:22,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/dc7b0df68777418d8a6efc80e8ea95df to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/dc7b0df68777418d8a6efc80e8ea95df 2024-11-12T00:54:22,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/a91eccadcbf44278bdbea0511ee3ae40 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/a91eccadcbf44278bdbea0511ee3ae40 2024-11-12T00:54:22,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/63c4a59f03374dddb1877bebf5624918 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/63c4a59f03374dddb1877bebf5624918 2024-11-12T00:54:22,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b3ef2c7a0cf949e89a99a19556d7d9d2 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b3ef2c7a0cf949e89a99a19556d7d9d2 2024-11-12T00:54:22,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/6c8781923b444095bcd31f35adcf17e5 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/6c8781923b444095bcd31f35adcf17e5 2024-11-12T00:54:22,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/81fc997e00354e3495c8646abf384a75 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/81fc997e00354e3495c8646abf384a75 2024-11-12T00:54:22,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b0b0b01cfebf42648aa349162b3602a9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b0b0b01cfebf42648aa349162b3602a9 2024-11-12T00:54:22,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7667e257b8ef428d95ff52464c164cb4 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7667e257b8ef428d95ff52464c164cb4 2024-11-12T00:54:22,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/5ff0ff789cbc433ab50d7158b8b4ead4 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/5ff0ff789cbc433ab50d7158b8b4ead4 2024-11-12T00:54:22,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/6529a82fd4a845d5a08a1363e8e9e2c5 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/6529a82fd4a845d5a08a1363e8e9e2c5 2024-11-12T00:54:22,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/a2c6c1198a6b44b1bdc64bff121df735 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/a2c6c1198a6b44b1bdc64bff121df735 2024-11-12T00:54:22,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b4221fa919994253a0b758781a445f94 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/b4221fa919994253a0b758781a445f94 2024-11-12T00:54:22,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7b9fbc2a17b84bd59aa56da0b0d20835 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7b9fbc2a17b84bd59aa56da0b0d20835 2024-11-12T00:54:22,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/cd4df67709d1402a9151b8156620f2b7 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/cd4df67709d1402a9151b8156620f2b7 2024-11-12T00:54:22,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8692685d9acf4d8e83a185f59144e706 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8692685d9acf4d8e83a185f59144e706 2024-11-12T00:54:22,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/049880fa40bf40779bf7450dc1e6c22d to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/049880fa40bf40779bf7450dc1e6c22d 2024-11-12T00:54:22,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7cff125bb0234640b13080906355b854 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/7cff125bb0234640b13080906355b854 2024-11-12T00:54:22,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/bb1928b77c3d4c8c93026549187d7b1e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/bb1928b77c3d4c8c93026549187d7b1e 2024-11-12T00:54:22,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/c9531cd40bb84f73879a9fee4348c03a to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/c9531cd40bb84f73879a9fee4348c03a 2024-11-12T00:54:22,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/be1de6f289794b95b7701e47f5e72fbe to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/be1de6f289794b95b7701e47f5e72fbe 2024-11-12T00:54:22,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/53ab977fac294beea903c8ac4425dacf to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/53ab977fac294beea903c8ac4425dacf 2024-11-12T00:54:22,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/310ea9f0d52d461b93a8e7ee63d962cd to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/310ea9f0d52d461b93a8e7ee63d962cd 2024-11-12T00:54:22,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/219873cc59574275a1e14a4ed4962a5f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/219873cc59574275a1e14a4ed4962a5f 2024-11-12T00:54:22,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/050a2a3925f44ef28243cf168a84bec0 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/050a2a3925f44ef28243cf168a84bec0 2024-11-12T00:54:22,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8343dd93d5cf42d5b3c84d7501c3d409 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8343dd93d5cf42d5b3c84d7501c3d409 2024-11-12T00:54:22,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/baf4edf22e3f445cb2cbd9fd2455a53e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/baf4edf22e3f445cb2cbd9fd2455a53e 2024-11-12T00:54:22,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/0bbb82f6255846c2a6d5c24163f0fc88 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/0bbb82f6255846c2a6d5c24163f0fc88 2024-11-12T00:54:22,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8957c8e334564884b6a86e518a407cda to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/8957c8e334564884b6a86e518a407cda 2024-11-12T00:54:22,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/91a9fbfe55d243428ae521918176ee06 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/91a9fbfe55d243428ae521918176ee06 2024-11-12T00:54:22,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/14c3e9d9e43c4f9fae79083463d92962 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/14c3e9d9e43c4f9fae79083463d92962 2024-11-12T00:54:22,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/bc9a7ae1e05046f1bc42a262df232c65 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/bc9a7ae1e05046f1bc42a262df232c65 2024-11-12T00:54:22,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/5c5893b493a649c0be1fcc363da30de5 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/5c5893b493a649c0be1fcc363da30de5 2024-11-12T00:54:22,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/14054e439a0f4e379217252df1aea48c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/7867a159f5b541808f69061f296f9680, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/9055ecae73604821ade088b9cf3fe9fd, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/e17e292b467a49a3b649076ad28bbe48, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/016e4e7960d546a4bd26eb1739f1d9f3, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/e95ff23636ec484599897a22cd63f7c3, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/67753333e3b24f6882e9949cc4d5d33c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/5178d0f69ebe404c9429cdff909c9ed3, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/857e6c91b7c54d1dbc9e4431fa058934, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/36804d7c27724cc781e5e0171e0ecc9e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/1d1cc9ea7c154c89868aa3a39c05fdb7, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/57c191a984664c3b97a34e64ea00cd9d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/02ed8983038045e08c0409472d8d83ca, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/cbed5c097f5f49889cd5de7ac71f72f1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/0da2ce6aa1dd4450a0f334cd1739b103, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/263242d6d0dc44028e16200caba332eb, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a24db3d27e874361a1abdcea3fd03780, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/aee380c5d3bd47deb8c4169cb0d8dbfe, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/225e4392ee1742aba8726ea9c4f14b62, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/834c2f8be5df4b7b9fb6c0cef420f917, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/f571c88616ab42a28ee760bcc6d31343, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/9150547d60e7476ea8ccd7bb886e284d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/5668cd73cf33400691569016686b473a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a6439057b5834efb83a33d6e783544b6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/93f5d6fd5cb84f88bc98df9f96afe407, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/af462253ac6e4b2d9d935dba3ca9b804, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/09f371b985284dd19af3d5a622f1d0ea, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/6e67bd749dd64c4bb2f8043f7d114815, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/1a428da803fc4549b73a1af6f094b81e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/19efaf7dddbe481eb3562e45d5d1b586, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/4760b0bad68e4c6390d63dbd06c9bd96, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/613e110da615468690595c5adfb92a83] to archive 2024-11-12T00:54:22,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:54:22,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/14054e439a0f4e379217252df1aea48c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/14054e439a0f4e379217252df1aea48c 2024-11-12T00:54:22,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/7867a159f5b541808f69061f296f9680 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/7867a159f5b541808f69061f296f9680 2024-11-12T00:54:22,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/9055ecae73604821ade088b9cf3fe9fd to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/9055ecae73604821ade088b9cf3fe9fd 2024-11-12T00:54:22,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/e17e292b467a49a3b649076ad28bbe48 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/e17e292b467a49a3b649076ad28bbe48 2024-11-12T00:54:22,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/016e4e7960d546a4bd26eb1739f1d9f3 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/016e4e7960d546a4bd26eb1739f1d9f3 2024-11-12T00:54:22,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/e95ff23636ec484599897a22cd63f7c3 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/e95ff23636ec484599897a22cd63f7c3 2024-11-12T00:54:22,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/67753333e3b24f6882e9949cc4d5d33c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/67753333e3b24f6882e9949cc4d5d33c 2024-11-12T00:54:22,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/5178d0f69ebe404c9429cdff909c9ed3 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/5178d0f69ebe404c9429cdff909c9ed3 2024-11-12T00:54:22,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/857e6c91b7c54d1dbc9e4431fa058934 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/857e6c91b7c54d1dbc9e4431fa058934 2024-11-12T00:54:22,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/36804d7c27724cc781e5e0171e0ecc9e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/36804d7c27724cc781e5e0171e0ecc9e 2024-11-12T00:54:22,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/1d1cc9ea7c154c89868aa3a39c05fdb7 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/1d1cc9ea7c154c89868aa3a39c05fdb7 2024-11-12T00:54:22,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/57c191a984664c3b97a34e64ea00cd9d to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/57c191a984664c3b97a34e64ea00cd9d 2024-11-12T00:54:22,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/02ed8983038045e08c0409472d8d83ca to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/02ed8983038045e08c0409472d8d83ca 2024-11-12T00:54:22,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/cbed5c097f5f49889cd5de7ac71f72f1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/cbed5c097f5f49889cd5de7ac71f72f1 2024-11-12T00:54:22,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/0da2ce6aa1dd4450a0f334cd1739b103 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/0da2ce6aa1dd4450a0f334cd1739b103 2024-11-12T00:54:22,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/263242d6d0dc44028e16200caba332eb to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/263242d6d0dc44028e16200caba332eb 2024-11-12T00:54:22,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a24db3d27e874361a1abdcea3fd03780 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a24db3d27e874361a1abdcea3fd03780 2024-11-12T00:54:22,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/aee380c5d3bd47deb8c4169cb0d8dbfe to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/aee380c5d3bd47deb8c4169cb0d8dbfe 2024-11-12T00:54:22,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/225e4392ee1742aba8726ea9c4f14b62 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/225e4392ee1742aba8726ea9c4f14b62 2024-11-12T00:54:22,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/834c2f8be5df4b7b9fb6c0cef420f917 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/834c2f8be5df4b7b9fb6c0cef420f917 2024-11-12T00:54:22,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/f571c88616ab42a28ee760bcc6d31343 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/f571c88616ab42a28ee760bcc6d31343 2024-11-12T00:54:22,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/9150547d60e7476ea8ccd7bb886e284d to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/9150547d60e7476ea8ccd7bb886e284d 2024-11-12T00:54:22,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/5668cd73cf33400691569016686b473a to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/5668cd73cf33400691569016686b473a 2024-11-12T00:54:22,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a6439057b5834efb83a33d6e783544b6 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a6439057b5834efb83a33d6e783544b6 2024-11-12T00:54:22,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/93f5d6fd5cb84f88bc98df9f96afe407 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/93f5d6fd5cb84f88bc98df9f96afe407 2024-11-12T00:54:22,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/af462253ac6e4b2d9d935dba3ca9b804 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/af462253ac6e4b2d9d935dba3ca9b804 2024-11-12T00:54:22,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/09f371b985284dd19af3d5a622f1d0ea to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/09f371b985284dd19af3d5a622f1d0ea 2024-11-12T00:54:22,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/6e67bd749dd64c4bb2f8043f7d114815 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/6e67bd749dd64c4bb2f8043f7d114815 2024-11-12T00:54:22,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/1a428da803fc4549b73a1af6f094b81e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/1a428da803fc4549b73a1af6f094b81e 2024-11-12T00:54:22,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/19efaf7dddbe481eb3562e45d5d1b586 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/19efaf7dddbe481eb3562e45d5d1b586 2024-11-12T00:54:22,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/4760b0bad68e4c6390d63dbd06c9bd96 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/4760b0bad68e4c6390d63dbd06c9bd96 2024-11-12T00:54:22,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/613e110da615468690595c5adfb92a83 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/613e110da615468690595c5adfb92a83 2024-11-12T00:54:22,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/437487d892614748b9e216b082e90631, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/1df66fa9b90b4e61966f91c30a328e48, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/c858830a30584f70b8c2ba2152e0d4d9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9a478915c1184145831919ab233b83dc, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/016986bd53584419a7f29c20bdd6175c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/677a8b7483144770bd8e15ba95b8310a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/b0c48449177d4f9faaee82e7d9d6cde6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4ce2f51f77584ff3b8e27c52ca6e9296, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4d3259e61cdd479ba1a519fd7ab872d6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/915667c826384b03a6c6fcc9b562aba8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6ee5687a2a3f47e290341bb880af6fc1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b7c615b48bc485dbaedc622f4559bab, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6da33c2ac18646deb96e8d530cf8aaa1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9afbf2ebab8341e2aa06e3b873f62723, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3bd5a429ba5a4ab886b05db7d14c5332, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/f82ad5be7eee4d669d62b72ccd252ed9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b6135eee5034f7abf8e9bce600aee45, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3a5afbfe1d7f458784781c6da8c4f095, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/c303672a26ac43ee820eaef305bdfda5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9d70ed88f86843c098be111a9c17b464, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3133167ee5044785896419e442eb82b8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4a2ac573e5b9475588b0e46b94f288b9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/73fc10e0ba304f4e9b0b140d022fa9e6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/81ca7e9187df49c0a0c56803c30ba276, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6930679620e64f25bd959f3918ba4c21, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/297b8a76802949ffb50b30cf482e491b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/947200d29a334d3daf88ff2913f0a27f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/5f29ad508f0548c2870ac1c337e1e0b5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/cbd5ea5bf0df4fe3a7ea599533853f75, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b6589a9b9bc4062848484201798494b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/2116f167bba7446099849a6b30d62ae5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/1a2682626abb4789b77c699ea7f5efe0] to archive 2024-11-12T00:54:22,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:54:22,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/437487d892614748b9e216b082e90631 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/437487d892614748b9e216b082e90631 2024-11-12T00:54:22,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/1df66fa9b90b4e61966f91c30a328e48 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/1df66fa9b90b4e61966f91c30a328e48 2024-11-12T00:54:22,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/c858830a30584f70b8c2ba2152e0d4d9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/c858830a30584f70b8c2ba2152e0d4d9 2024-11-12T00:54:22,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9a478915c1184145831919ab233b83dc to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9a478915c1184145831919ab233b83dc 2024-11-12T00:54:22,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/016986bd53584419a7f29c20bdd6175c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/016986bd53584419a7f29c20bdd6175c 2024-11-12T00:54:22,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/677a8b7483144770bd8e15ba95b8310a to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/677a8b7483144770bd8e15ba95b8310a 2024-11-12T00:54:22,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/b0c48449177d4f9faaee82e7d9d6cde6 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/b0c48449177d4f9faaee82e7d9d6cde6 2024-11-12T00:54:22,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4ce2f51f77584ff3b8e27c52ca6e9296 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4ce2f51f77584ff3b8e27c52ca6e9296 2024-11-12T00:54:22,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4d3259e61cdd479ba1a519fd7ab872d6 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4d3259e61cdd479ba1a519fd7ab872d6 2024-11-12T00:54:22,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/915667c826384b03a6c6fcc9b562aba8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/915667c826384b03a6c6fcc9b562aba8 2024-11-12T00:54:22,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6ee5687a2a3f47e290341bb880af6fc1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6ee5687a2a3f47e290341bb880af6fc1 2024-11-12T00:54:22,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b7c615b48bc485dbaedc622f4559bab to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b7c615b48bc485dbaedc622f4559bab 2024-11-12T00:54:22,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6da33c2ac18646deb96e8d530cf8aaa1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6da33c2ac18646deb96e8d530cf8aaa1 2024-11-12T00:54:22,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9afbf2ebab8341e2aa06e3b873f62723 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9afbf2ebab8341e2aa06e3b873f62723 2024-11-12T00:54:22,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3bd5a429ba5a4ab886b05db7d14c5332 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3bd5a429ba5a4ab886b05db7d14c5332 2024-11-12T00:54:22,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/f82ad5be7eee4d669d62b72ccd252ed9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/f82ad5be7eee4d669d62b72ccd252ed9 2024-11-12T00:54:22,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b6135eee5034f7abf8e9bce600aee45 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b6135eee5034f7abf8e9bce600aee45 2024-11-12T00:54:22,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3a5afbfe1d7f458784781c6da8c4f095 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3a5afbfe1d7f458784781c6da8c4f095 2024-11-12T00:54:22,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/c303672a26ac43ee820eaef305bdfda5 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/c303672a26ac43ee820eaef305bdfda5 2024-11-12T00:54:22,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9d70ed88f86843c098be111a9c17b464 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/9d70ed88f86843c098be111a9c17b464 2024-11-12T00:54:22,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3133167ee5044785896419e442eb82b8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/3133167ee5044785896419e442eb82b8 2024-11-12T00:54:22,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4a2ac573e5b9475588b0e46b94f288b9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/4a2ac573e5b9475588b0e46b94f288b9 2024-11-12T00:54:22,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/73fc10e0ba304f4e9b0b140d022fa9e6 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/73fc10e0ba304f4e9b0b140d022fa9e6 2024-11-12T00:54:22,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/81ca7e9187df49c0a0c56803c30ba276 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/81ca7e9187df49c0a0c56803c30ba276 2024-11-12T00:54:22,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6930679620e64f25bd959f3918ba4c21 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/6930679620e64f25bd959f3918ba4c21 2024-11-12T00:54:22,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/297b8a76802949ffb50b30cf482e491b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/297b8a76802949ffb50b30cf482e491b 2024-11-12T00:54:22,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/947200d29a334d3daf88ff2913f0a27f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/947200d29a334d3daf88ff2913f0a27f 2024-11-12T00:54:22,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/5f29ad508f0548c2870ac1c337e1e0b5 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/5f29ad508f0548c2870ac1c337e1e0b5 2024-11-12T00:54:22,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/cbd5ea5bf0df4fe3a7ea599533853f75 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/cbd5ea5bf0df4fe3a7ea599533853f75 2024-11-12T00:54:22,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b6589a9b9bc4062848484201798494b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/7b6589a9b9bc4062848484201798494b 2024-11-12T00:54:22,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/2116f167bba7446099849a6b30d62ae5 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/2116f167bba7446099849a6b30d62ae5 2024-11-12T00:54:22,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/1a2682626abb4789b77c699ea7f5efe0 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/1a2682626abb4789b77c699ea7f5efe0 2024-11-12T00:54:22,108 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/recovered.edits/481.seqid, newMaxSeqId=481, maxSeqId=1 2024-11-12T00:54:22,109 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. 2024-11-12T00:54:22,109 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for eacf29d404b7a11371b9ac1286c14c2b: Waiting for close lock at 1731372860761Running coprocessor pre-close hooks at 1731372860761Disabling compacts and flushes for region at 1731372860761Disabling writes for close at 1731372860761Obtaining lock to block concurrent updates at 1731372860762 (+1 ms)Preparing flush snapshotting stores in eacf29d404b7a11371b9ac1286c14c2b at 1731372860762Finished memstore snapshotting TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731372860762Flushing stores of TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b. at 1731372860763 (+1 ms)Flushing eacf29d404b7a11371b9ac1286c14c2b/A: creating writer at 1731372860764 (+1 ms)Flushing eacf29d404b7a11371b9ac1286c14c2b/A: appending metadata at 1731372860769 (+5 ms)Flushing eacf29d404b7a11371b9ac1286c14c2b/A: closing flushed file at 1731372860769Flushing eacf29d404b7a11371b9ac1286c14c2b/B: creating writer at 1731372861178 (+409 ms)Flushing eacf29d404b7a11371b9ac1286c14c2b/B: appending metadata at 1731372861180 (+2 ms)Flushing eacf29d404b7a11371b9ac1286c14c2b/B: closing flushed file at 1731372861181 (+1 ms)Flushing eacf29d404b7a11371b9ac1286c14c2b/C: creating writer at 1731372861588 (+407 ms)Flushing eacf29d404b7a11371b9ac1286c14c2b/C: appending metadata at 1731372861591 (+3 ms)Flushing eacf29d404b7a11371b9ac1286c14c2b/C: closing flushed file at 1731372861591Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30880179: reopening flushed file at 1731372862006 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@405ca975: reopening flushed file at 1731372862012 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7642c942: reopening flushed file at 1731372862018 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for eacf29d404b7a11371b9ac1286c14c2b in 1263ms, sequenceid=478, compaction requested=true at 1731372862024 (+6 ms)Writing region close event to WAL at 1731372862104 (+80 ms)Running coprocessor post-close hooks at 1731372862109 (+5 ms)Closed at 1731372862109 2024-11-12T00:54:22,112 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:22,112 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=eacf29d404b7a11371b9ac1286c14c2b, regionState=CLOSED 2024-11-12T00:54:22,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:54:22,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-12T00:54:22,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure eacf29d404b7a11371b9ac1286c14c2b, server=6928b69dca1b,33953,1731372765559 in 1.5080 sec 2024-11-12T00:54:22,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-12T00:54:22,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=eacf29d404b7a11371b9ac1286c14c2b, UNASSIGN in 1.5130 sec 2024-11-12T00:54:22,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-12T00:54:22,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5160 sec 2024-11-12T00:54:22,121 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372862121"}]},"ts":"1731372862121"} 2024-11-12T00:54:22,122 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T00:54:22,122 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T00:54:22,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5280 sec 2024-11-12T00:54:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T00:54:22,757 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:22,758 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T00:54:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:22,762 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:22,764 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=80, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T00:54:22,768 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:22,772 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/recovered.edits] 2024-11-12T00:54:22,776 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/2752580f13e24f7084aaf7bf9c7228dc to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/2752580f13e24f7084aaf7bf9c7228dc 2024-11-12T00:54:22,778 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/6775c14c2a004c70857918b599c8e6cc to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/6775c14c2a004c70857918b599c8e6cc 2024-11-12T00:54:22,780 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/ac443a52c5e34578b35ce8d829165dc7 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/A/ac443a52c5e34578b35ce8d829165dc7 2024-11-12T00:54:22,784 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/771c60f853524ad7a31439e1e02ffae0 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/771c60f853524ad7a31439e1e02ffae0 2024-11-12T00:54:22,786 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/9b9fe71cdf974258b637e48691e7329a to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/9b9fe71cdf974258b637e48691e7329a 2024-11-12T00:54:22,788 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a549a83fcf4b49d999c6659cd00e4a4d to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/B/a549a83fcf4b49d999c6659cd00e4a4d 2024-11-12T00:54:22,791 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/0c9434a8dd204b6ebb21db2a7da24244 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/0c9434a8dd204b6ebb21db2a7da24244 2024-11-12T00:54:22,793 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/37d5e874a94f4f01b1e2397dfd1bfe20 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/37d5e874a94f4f01b1e2397dfd1bfe20 2024-11-12T00:54:22,795 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/cf31e1c10e7546ad9a1cc08e91b72e5a to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/C/cf31e1c10e7546ad9a1cc08e91b72e5a 2024-11-12T00:54:22,799 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/recovered.edits/481.seqid to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b/recovered.edits/481.seqid 2024-11-12T00:54:22,800 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/eacf29d404b7a11371b9ac1286c14c2b 2024-11-12T00:54:22,801 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T00:54:22,804 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=80, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:22,807 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T00:54:22,809 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T00:54:22,810 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=80, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:22,810 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T00:54:22,810 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731372862810"}]},"ts":"9223372036854775807"} 2024-11-12T00:54:22,812 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T00:54:22,812 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => eacf29d404b7a11371b9ac1286c14c2b, NAME => 'TestAcidGuarantees,,1731372833808.eacf29d404b7a11371b9ac1286c14c2b.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T00:54:22,812 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T00:54:22,812 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731372862812"}]},"ts":"9223372036854775807"} 2024-11-12T00:54:22,813 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T00:54:22,814 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=80, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:22,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-11-12T00:54:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T00:54:22,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T00:54:22,882 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:22,892 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=247 (was 248), OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=450 (was 430) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5229 (was 4817) - AvailableMemoryMB LEAK? - 2024-11-12T00:54:22,901 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=450, ProcessCount=11, AvailableMemoryMB=5229 2024-11-12T00:54:22,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T00:54:22,903 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T00:54:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:22,905 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T00:54:22,906 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:22,906 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 81 2024-11-12T00:54:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-12T00:54:22,907 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T00:54:22,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742173_1349 (size=1061) 2024-11-12T00:54:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-12T00:54:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-12T00:54:23,318 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6 2024-11-12T00:54:23,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742174_1350 (size=53) 2024-11-12T00:54:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-12T00:54:23,730 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:54:23,731 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 58142a01560cfc2d88d5988d1ba12ec1, disabling compactions & flushes 2024-11-12T00:54:23,731 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:23,731 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:23,731 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. after waiting 0 ms 2024-11-12T00:54:23,731 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:23,731 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:23,731 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 58142a01560cfc2d88d5988d1ba12ec1: Waiting for close lock at 1731372863731Disabling compacts and flushes for region at 1731372863731Disabling writes for close at 1731372863731Writing region close event to WAL at 1731372863731Closed at 1731372863731 2024-11-12T00:54:23,734 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T00:54:23,735 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731372863734"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731372863734"}]},"ts":"1731372863734"} 2024-11-12T00:54:23,738 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T00:54:23,739 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T00:54:23,740 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372863739"}]},"ts":"1731372863739"} 2024-11-12T00:54:23,742 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T00:54:23,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58142a01560cfc2d88d5988d1ba12ec1, ASSIGN}] 2024-11-12T00:54:23,744 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58142a01560cfc2d88d5988d1ba12ec1, ASSIGN 2024-11-12T00:54:23,745 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58142a01560cfc2d88d5988d1ba12ec1, ASSIGN; state=OFFLINE, location=6928b69dca1b,33953,1731372765559; forceNewPlan=false, retain=false 2024-11-12T00:54:23,896 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=58142a01560cfc2d88d5988d1ba12ec1, regionState=OPENING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:23,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58142a01560cfc2d88d5988d1ba12ec1, ASSIGN because future has completed 2024-11-12T00:54:23,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:54:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-12T00:54:24,058 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:24,059 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7752): Opening region: {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} 2024-11-12T00:54:24,059 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:24,059 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:54:24,059 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7794): checking encryption for 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:24,060 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7797): checking classloading for 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:24,062 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:24,064 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:54:24,064 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58142a01560cfc2d88d5988d1ba12ec1 columnFamilyName A 2024-11-12T00:54:24,064 DEBUG [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:24,065 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.HStore(327): Store=58142a01560cfc2d88d5988d1ba12ec1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:54:24,065 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:24,067 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:54:24,067 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58142a01560cfc2d88d5988d1ba12ec1 columnFamilyName B 2024-11-12T00:54:24,067 DEBUG [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:24,068 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.HStore(327): Store=58142a01560cfc2d88d5988d1ba12ec1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:54:24,068 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:24,070 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:54:24,070 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58142a01560cfc2d88d5988d1ba12ec1 columnFamilyName C 2024-11-12T00:54:24,070 DEBUG [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:24,071 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.HStore(327): Store=58142a01560cfc2d88d5988d1ba12ec1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:54:24,071 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:24,072 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1038): replaying wal for 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:24,072 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:24,072 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:24,073 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1048): stopping wal replay for 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:24,073 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1060): Cleaning up temporary data for 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:24,074 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T00:54:24,075 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): writing seq id for 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:24,077 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T00:54:24,077 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1114): Opened 58142a01560cfc2d88d5988d1ba12ec1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61961495, jitterRate=-0.07670177519321442}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T00:54:24,077 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:24,077 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1006): Region open journal for 58142a01560cfc2d88d5988d1ba12ec1: Running coprocessor pre-open hook at 1731372864060Writing region info on filesystem at 1731372864060Initializing all the Stores at 1731372864061 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372864061Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372864061Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372864061Cleaning up temporary data from old regions at 1731372864073 (+12 ms)Running coprocessor post-open hooks at 1731372864077 (+4 ms)Region opened successfully at 1731372864077 2024-11-12T00:54:24,078 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., pid=83, masterSystemTime=1731372864054 2024-11-12T00:54:24,080 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:24,080 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:24,080 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=58142a01560cfc2d88d5988d1ba12ec1, regionState=OPEN, openSeqNum=2, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:24,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:54:24,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-12T00:54:24,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure 58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 in 182 msec 2024-11-12T00:54:24,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-12T00:54:24,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58142a01560cfc2d88d5988d1ba12ec1, ASSIGN in 341 msec 2024-11-12T00:54:24,085 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T00:54:24,086 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372864085"}]},"ts":"1731372864085"} 2024-11-12T00:54:24,087 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T00:54:24,088 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T00:54:24,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-11-12T00:54:25,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-12T00:54:25,058 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:25,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ab9835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:25,060 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:25,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:25,064 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@629f3775, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:25,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:25,067 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a55aeaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:25,068 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:25,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:25,071 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T00:54:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 6928b69dca1b,45925,1731372764943 2024-11-12T00:54:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@73752795 2024-11-12T00:54:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T00:54:25,073 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T00:54:25,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T00:54:25,074 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T00:54:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:25,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742175_1351 (size=1097) 2024-11-12T00:54:25,490 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T00:54:25,490 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T00:54:25,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T00:54:25,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58142a01560cfc2d88d5988d1ba12ec1, REOPEN/MOVE}] 2024-11-12T00:54:25,500 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58142a01560cfc2d88d5988d1ba12ec1, REOPEN/MOVE 2024-11-12T00:54:25,501 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=58142a01560cfc2d88d5988d1ba12ec1, regionState=CLOSING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:25,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58142a01560cfc2d88d5988d1ba12ec1, REOPEN/MOVE because future has completed 2024-11-12T00:54:25,504 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T00:54:25,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:54:25,658 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:25,658 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T00:54:25,658 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 58142a01560cfc2d88d5988d1ba12ec1, disabling compactions & flushes 2024-11-12T00:54:25,658 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:25,658 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:25,658 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. after waiting 0 ms 2024-11-12T00:54:25,658 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:25,666 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T00:54:25,667 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:25,667 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 58142a01560cfc2d88d5988d1ba12ec1: Waiting for close lock at 1731372865658Running coprocessor pre-close hooks at 1731372865658Disabling compacts and flushes for region at 1731372865658Disabling writes for close at 1731372865658Writing region close event to WAL at 1731372865659 (+1 ms)Running coprocessor post-close hooks at 1731372865667 (+8 ms)Closed at 1731372865667 2024-11-12T00:54:25,667 WARN [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionServer(3299): Not adding moved region record: 58142a01560cfc2d88d5988d1ba12ec1 to self. 2024-11-12T00:54:25,670 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:25,670 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=58142a01560cfc2d88d5988d1ba12ec1, regionState=CLOSED 2024-11-12T00:54:25,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:54:25,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-12T00:54:25,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 in 168 msec 2024-11-12T00:54:25,674 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58142a01560cfc2d88d5988d1ba12ec1, REOPEN/MOVE; state=CLOSED, location=6928b69dca1b,33953,1731372765559; forceNewPlan=false, retain=true 2024-11-12T00:54:25,825 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=58142a01560cfc2d88d5988d1ba12ec1, regionState=OPENING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:25,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58142a01560cfc2d88d5988d1ba12ec1, REOPEN/MOVE because future has completed 2024-11-12T00:54:25,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:54:25,989 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:25,990 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} 2024-11-12T00:54:25,991 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:25,991 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:54:25,991 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:25,991 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:25,993 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:25,994 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:54:25,994 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58142a01560cfc2d88d5988d1ba12ec1 columnFamilyName A 2024-11-12T00:54:25,996 DEBUG [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:25,997 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.HStore(327): Store=58142a01560cfc2d88d5988d1ba12ec1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:54:25,997 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:25,998 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:54:25,998 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58142a01560cfc2d88d5988d1ba12ec1 columnFamilyName B 2024-11-12T00:54:25,998 DEBUG [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:25,999 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.HStore(327): Store=58142a01560cfc2d88d5988d1ba12ec1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:54:25,999 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:26,000 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:54:26,000 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58142a01560cfc2d88d5988d1ba12ec1 columnFamilyName C 2024-11-12T00:54:26,000 DEBUG [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:26,001 INFO [StoreOpener-58142a01560cfc2d88d5988d1ba12ec1-1 {}] regionserver.HStore(327): Store=58142a01560cfc2d88d5988d1ba12ec1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:54:26,001 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:26,001 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:26,002 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:26,003 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:26,004 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:26,004 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:26,005 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T00:54:26,006 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:26,006 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened 58142a01560cfc2d88d5988d1ba12ec1; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70322367, jitterRate=0.047884926199913025}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T00:54:26,006 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:26,007 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for 58142a01560cfc2d88d5988d1ba12ec1: Running coprocessor pre-open hook at 1731372865992Writing region info on filesystem at 1731372865992Initializing all the Stores at 1731372865993 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372865993Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372865993Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372865993Cleaning up temporary data from old regions at 1731372866004 (+11 ms)Running coprocessor post-open hooks at 1731372866006 (+2 ms)Region opened successfully at 1731372866007 (+1 ms) 2024-11-12T00:54:26,008 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., pid=88, masterSystemTime=1731372865982 2024-11-12T00:54:26,009 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:26,009 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:26,009 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=58142a01560cfc2d88d5988d1ba12ec1, regionState=OPEN, openSeqNum=5, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:54:26,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=86 2024-11-12T00:54:26,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 in 183 msec 2024-11-12T00:54:26,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-12T00:54:26,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58142a01560cfc2d88d5988d1ba12ec1, REOPEN/MOVE in 513 msec 2024-11-12T00:54:26,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-12T00:54:26,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 519 msec 2024-11-12T00:54:26,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 941 msec 2024-11-12T00:54:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T00:54:26,018 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:26,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11317fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,019 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:26,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:26,019 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@327a2304, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:26,020 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@648ef73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:26,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:26,022 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d7c306a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:26,023 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:26,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15b7f0e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:26,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:26,025 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32fbd737, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:26,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:26,026 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5484d8cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:26,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:26,027 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7be457, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:26,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:26,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:26,028 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:26,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55f29911, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,030 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:26,030 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:26,030 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12562b56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:26,032 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@460944ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:26,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:26,033 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48d13986, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:26,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:26,035 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:26,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1398db80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:26,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:26,036 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2a74ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:26,038 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a9aa0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,039 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:26,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:26,039 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ff7e4f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:26,040 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf10e36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:26,042 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:26,042 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28f2a215, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:26,044 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f6a7f63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,045 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:26,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:26,046 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d2a291f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:26,047 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb486d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:26,048 DEBUG [Thread-1615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:26,048 DEBUG [Thread-1609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:26,048 DEBUG [Thread-1600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:26,048 DEBUG [Thread-1613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:26,049 DEBUG [Thread-1611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:26,049 DEBUG [Thread-1596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:26,049 DEBUG [Thread-1602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:26,049 DEBUG [Thread-1598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:26,050 DEBUG [Thread-1604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:26,050 DEBUG [Thread-1602 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:26,050 DEBUG [Thread-1607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:26,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:26,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:26,051 DEBUG [Thread-1602 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:26,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:26,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:26,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:26,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:26,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:26,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:26,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:26,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:26,053 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:54:26,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:54:26,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-12T00:54:26,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:54:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:54:26,056 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:54:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T00:54:26,056 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:54:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:54:26,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:54:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:54:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:54:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:54:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:54:26,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:54:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:54:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:26,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T00:54:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:26,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127ef197c0904741a4b8e2bc87aed2c4f8_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_0/A:col10/1731372866066/Put/seqid=0 2024-11-12T00:54:26,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742176_1352 (size=17034) 2024-11-12T00:54:26,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372926099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372926104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372926105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372926112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372926114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T00:54:26,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-12T00:54:26,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:26,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:26,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:26,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372926225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372926225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372926225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372926226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372926226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-12T00:54:26,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:26,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:26,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:26,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T00:54:26,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372926435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372926436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372926436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372926436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372926436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,514 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:26,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-12T00:54:26,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:26,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:26,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:26,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,517 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127ef197c0904741a4b8e2bc87aed2c4f8_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127ef197c0904741a4b8e2bc87aed2c4f8_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,518 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/3749c4fd64d84b788925b15262425af4, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:26,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/3749c4fd64d84b788925b15262425af4 is 175, key is test_row_0/A:col10/1731372866066/Put/seqid=0 2024-11-12T00:54:26,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742177_1353 (size=48139) 2024-11-12T00:54:26,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-12T00:54:26,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:26,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:26,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:26,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T00:54:26,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372926745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372926745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372926756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372926756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:26,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372926756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:26,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:26,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:26,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-12T00:54:26,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:26,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:26,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:26,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,921 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/3749c4fd64d84b788925b15262425af4 2024-11-12T00:54:26,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/2e150201ffef458f9e3f8e7875817fa6 is 50, key is test_row_0/B:col10/1731372866066/Put/seqid=0 2024-11-12T00:54:26,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742178_1354 (size=12001) 2024-11-12T00:54:26,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-12T00:54:26,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:26,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:26,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:26,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:26,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-12T00:54:27,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:27,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:27,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:27,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T00:54:27,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372927256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:27,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372927256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:27,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372927265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:27,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372927265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:27,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372927265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:27,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-12T00:54:27,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:27,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:27,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:27,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/2e150201ffef458f9e3f8e7875817fa6 2024-11-12T00:54:27,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/96da969663aa461c936e7c8cd2ba371c is 50, key is test_row_0/C:col10/1731372866066/Put/seqid=0 2024-11-12T00:54:27,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742179_1355 (size=12001) 2024-11-12T00:54:27,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-12T00:54:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:27,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-12T00:54:27,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:27,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:27,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:27,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,633 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T00:54:27,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-12T00:54:27,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:27,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:27,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:27,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:27,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/96da969663aa461c936e7c8cd2ba371c 2024-11-12T00:54:27,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/3749c4fd64d84b788925b15262425af4 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/3749c4fd64d84b788925b15262425af4 2024-11-12T00:54:27,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/3749c4fd64d84b788925b15262425af4, entries=250, sequenceid=17, filesize=47.0 K 2024-11-12T00:54:27,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/2e150201ffef458f9e3f8e7875817fa6 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/2e150201ffef458f9e3f8e7875817fa6 2024-11-12T00:54:27,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/2e150201ffef458f9e3f8e7875817fa6, entries=150, sequenceid=17, filesize=11.7 K 2024-11-12T00:54:27,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/96da969663aa461c936e7c8cd2ba371c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/96da969663aa461c936e7c8cd2ba371c 2024-11-12T00:54:27,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/96da969663aa461c936e7c8cd2ba371c, entries=150, sequenceid=17, filesize=11.7 K 2024-11-12T00:54:27,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 58142a01560cfc2d88d5988d1ba12ec1 in 1730ms, sequenceid=17, compaction requested=false 2024-11-12T00:54:27,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:27,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-12T00:54:27,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:27,891 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T00:54:27,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:27,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:27,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:27,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:27,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:27,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:27,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c71eb4c8cf024db4b667ffbab728585b_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_0/A:col10/1731372866088/Put/seqid=0 2024-11-12T00:54:27,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742180_1356 (size=12154) 2024-11-12T00:54:27,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:27,906 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c71eb4c8cf024db4b667ffbab728585b_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c71eb4c8cf024db4b667ffbab728585b_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:27,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/17fb0abd6ce24188a28caaa9838c7131, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:27,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/17fb0abd6ce24188a28caaa9838c7131 is 175, key is test_row_0/A:col10/1731372866088/Put/seqid=0 2024-11-12T00:54:27,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742181_1357 (size=30955) 2024-11-12T00:54:27,912 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/17fb0abd6ce24188a28caaa9838c7131 2024-11-12T00:54:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/ef5980eeb02645e49c19fa5925601058 is 50, key is test_row_0/B:col10/1731372866088/Put/seqid=0 2024-11-12T00:54:27,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742182_1358 (size=12001) 2024-11-12T00:54:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T00:54:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:28,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:28,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372928292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372928294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372928295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372928297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372928298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,322 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/ef5980eeb02645e49c19fa5925601058 2024-11-12T00:54:28,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/dd35c26a3c024071b739244ec434e66b is 50, key is test_row_0/C:col10/1731372866088/Put/seqid=0 2024-11-12T00:54:28,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742183_1359 (size=12001) 2024-11-12T00:54:28,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372928415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372928416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372928416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372928416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372928416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372928625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372928626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372928636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372928636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372928636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,732 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/dd35c26a3c024071b739244ec434e66b 2024-11-12T00:54:28,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/17fb0abd6ce24188a28caaa9838c7131 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/17fb0abd6ce24188a28caaa9838c7131 2024-11-12T00:54:28,742 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/17fb0abd6ce24188a28caaa9838c7131, entries=150, sequenceid=36, filesize=30.2 K 2024-11-12T00:54:28,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/ef5980eeb02645e49c19fa5925601058 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ef5980eeb02645e49c19fa5925601058 2024-11-12T00:54:28,746 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ef5980eeb02645e49c19fa5925601058, entries=150, sequenceid=36, filesize=11.7 K 2024-11-12T00:54:28,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/dd35c26a3c024071b739244ec434e66b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/dd35c26a3c024071b739244ec434e66b 2024-11-12T00:54:28,750 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/dd35c26a3c024071b739244ec434e66b, entries=150, sequenceid=36, filesize=11.7 K 2024-11-12T00:54:28,751 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 58142a01560cfc2d88d5988d1ba12ec1 in 860ms, sequenceid=36, compaction requested=false 2024-11-12T00:54:28,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:28,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:28,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-12T00:54:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-12T00:54:28,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-12T00:54:28,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6950 sec 2024-11-12T00:54:28,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.7010 sec 2024-11-12T00:54:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:28,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T00:54:28,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:28,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:28,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:28,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:28,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:28,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:28,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112caf28e64b05a4959b4ce8f96abd484c0_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_0/A:col10/1731372868941/Put/seqid=0 2024-11-12T00:54:28,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742184_1360 (size=14594) 2024-11-12T00:54:28,956 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:28,967 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112caf28e64b05a4959b4ce8f96abd484c0_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112caf28e64b05a4959b4ce8f96abd484c0_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:28,968 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/adcf4b52e80747fea9e613b30e8b3390, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:28,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/adcf4b52e80747fea9e613b30e8b3390 is 175, key is test_row_0/A:col10/1731372868941/Put/seqid=0 2024-11-12T00:54:28,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742185_1361 (size=39549) 2024-11-12T00:54:28,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372928978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372928987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:28,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372928987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372928989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372928989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372929095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372929095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372929106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372929106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372929106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372929305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372929305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372929326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372929326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372929326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,376 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/adcf4b52e80747fea9e613b30e8b3390 2024-11-12T00:54:29,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/7e5c2aaed0b346cf9ed5afac583f1e0b is 50, key is test_row_0/B:col10/1731372868941/Put/seqid=0 2024-11-12T00:54:29,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742186_1362 (size=12001) 2024-11-12T00:54:29,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372929625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372929625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372929646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372929646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372929646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:29,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/7e5c2aaed0b346cf9ed5afac583f1e0b 2024-11-12T00:54:29,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/948514d1e56d40dcbaebaf1762addfa2 is 50, key is test_row_0/C:col10/1731372868941/Put/seqid=0 2024-11-12T00:54:29,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742187_1363 (size=12001) 2024-11-12T00:54:29,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/948514d1e56d40dcbaebaf1762addfa2 2024-11-12T00:54:29,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/adcf4b52e80747fea9e613b30e8b3390 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/adcf4b52e80747fea9e613b30e8b3390 2024-11-12T00:54:29,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/adcf4b52e80747fea9e613b30e8b3390, entries=200, sequenceid=50, filesize=38.6 K 2024-11-12T00:54:29,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/7e5c2aaed0b346cf9ed5afac583f1e0b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/7e5c2aaed0b346cf9ed5afac583f1e0b 2024-11-12T00:54:29,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/7e5c2aaed0b346cf9ed5afac583f1e0b, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T00:54:29,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/948514d1e56d40dcbaebaf1762addfa2 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/948514d1e56d40dcbaebaf1762addfa2 2024-11-12T00:54:29,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/948514d1e56d40dcbaebaf1762addfa2, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T00:54:29,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 58142a01560cfc2d88d5988d1ba12ec1 in 868ms, sequenceid=50, compaction requested=true 2024-11-12T00:54:29,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:29,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:54:29,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:29,810 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:29,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:54:29,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:29,810 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:29,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:54:29,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:29,811 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:29,811 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:29,811 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/A is initiating minor compaction (all files) 2024-11-12T00:54:29,811 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/B is initiating minor compaction (all files) 2024-11-12T00:54:29,811 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/A in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:29,811 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/B in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:29,811 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/2e150201ffef458f9e3f8e7875817fa6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ef5980eeb02645e49c19fa5925601058, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/7e5c2aaed0b346cf9ed5afac583f1e0b] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=35.2 K 2024-11-12T00:54:29,811 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/3749c4fd64d84b788925b15262425af4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/17fb0abd6ce24188a28caaa9838c7131, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/adcf4b52e80747fea9e613b30e8b3390] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=115.9 K 2024-11-12T00:54:29,811 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:29,811 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/3749c4fd64d84b788925b15262425af4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/17fb0abd6ce24188a28caaa9838c7131, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/adcf4b52e80747fea9e613b30e8b3390] 2024-11-12T00:54:29,812 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e150201ffef458f9e3f8e7875817fa6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731372866063 2024-11-12T00:54:29,813 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting ef5980eeb02645e49c19fa5925601058, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731372866088 2024-11-12T00:54:29,813 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3749c4fd64d84b788925b15262425af4, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731372866063 2024-11-12T00:54:29,813 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e5c2aaed0b346cf9ed5afac583f1e0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731372868295 2024-11-12T00:54:29,820 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17fb0abd6ce24188a28caaa9838c7131, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731372866088 2024-11-12T00:54:29,820 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#B#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:29,820 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting adcf4b52e80747fea9e613b30e8b3390, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731372868289 2024-11-12T00:54:29,821 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/ae4f3f167a134a33b91cce0dd14c2213 is 50, key is test_row_0/B:col10/1731372868941/Put/seqid=0 2024-11-12T00:54:29,830 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:29,832 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112a7d838f91cc34109807c659c034ef309_58142a01560cfc2d88d5988d1ba12ec1 store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:29,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742188_1364 (size=12104) 2024-11-12T00:54:29,834 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112a7d838f91cc34109807c659c034ef309_58142a01560cfc2d88d5988d1ba12ec1, store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:29,834 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a7d838f91cc34109807c659c034ef309_58142a01560cfc2d88d5988d1ba12ec1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:29,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742189_1365 (size=4469) 2024-11-12T00:54:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:30,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T00:54:30,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:30,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:30,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:30,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:30,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:30,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:30,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b2f6240023424499a3f5c2863e4752b9_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_0/A:col10/1731372868985/Put/seqid=0 2024-11-12T00:54:30,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742190_1366 (size=14594) 2024-11-12T00:54:30,158 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:30,161 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b2f6240023424499a3f5c2863e4752b9_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b2f6240023424499a3f5c2863e4752b9_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:30,162 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/e54bdb845a424587b5d58739b84e73d8, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:30,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/e54bdb845a424587b5d58739b84e73d8 is 175, key is test_row_0/A:col10/1731372868985/Put/seqid=0 2024-11-12T00:54:30,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742191_1367 (size=39549) 2024-11-12T00:54:30,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372930172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372930172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372930172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372930174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372930175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T00:54:30,216 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:30,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:54:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-12T00:54:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T00:54:30,218 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:54:30,219 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:54:30,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:54:30,236 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/ae4f3f167a134a33b91cce0dd14c2213 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ae4f3f167a134a33b91cce0dd14c2213 2024-11-12T00:54:30,240 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/B of 58142a01560cfc2d88d5988d1ba12ec1 into ae4f3f167a134a33b91cce0dd14c2213(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:30,240 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:30,240 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/B, priority=13, startTime=1731372869810; duration=0sec 2024-11-12T00:54:30,240 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:30,240 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:B 2024-11-12T00:54:30,240 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:30,241 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:30,241 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/C is initiating minor compaction (all files) 2024-11-12T00:54:30,241 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/C in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:30,241 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/96da969663aa461c936e7c8cd2ba371c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/dd35c26a3c024071b739244ec434e66b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/948514d1e56d40dcbaebaf1762addfa2] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=35.2 K 2024-11-12T00:54:30,241 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 96da969663aa461c936e7c8cd2ba371c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731372866063 2024-11-12T00:54:30,241 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting dd35c26a3c024071b739244ec434e66b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731372866088 2024-11-12T00:54:30,242 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 948514d1e56d40dcbaebaf1762addfa2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731372868295 2024-11-12T00:54:30,243 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#A#compaction#313 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:30,243 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/0b6501314ceb4b3a8a836a61fa0067f7 is 175, key is test_row_0/A:col10/1731372868941/Put/seqid=0 2024-11-12T00:54:30,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742192_1368 (size=31058) 2024-11-12T00:54:30,251 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#C#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:30,251 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/eb6e2b318715456887e80c5eb9750321 is 50, key is test_row_0/C:col10/1731372868941/Put/seqid=0 2024-11-12T00:54:30,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742193_1369 (size=12104) 2024-11-12T00:54:30,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372930286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372930286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372930286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372930286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372930286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T00:54:30,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-12T00:54:30,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:30,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:30,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:30,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:30,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:30,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372930496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372930496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372930496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372930496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372930496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-12T00:54:30,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:30,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:30,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:30,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:30,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T00:54:30,565 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/e54bdb845a424587b5d58739b84e73d8 2024-11-12T00:54:30,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/aca884b3b04c495abd5f5779726cef14 is 50, key is test_row_0/B:col10/1731372868985/Put/seqid=0 2024-11-12T00:54:30,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742194_1370 (size=12001) 2024-11-12T00:54:30,652 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/0b6501314ceb4b3a8a836a61fa0067f7 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0b6501314ceb4b3a8a836a61fa0067f7 2024-11-12T00:54:30,656 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/A of 58142a01560cfc2d88d5988d1ba12ec1 into 0b6501314ceb4b3a8a836a61fa0067f7(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:30,656 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:30,656 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/A, priority=13, startTime=1731372869810; duration=0sec 2024-11-12T00:54:30,656 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:30,656 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:A 2024-11-12T00:54:30,658 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/eb6e2b318715456887e80c5eb9750321 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/eb6e2b318715456887e80c5eb9750321 2024-11-12T00:54:30,664 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/C of 58142a01560cfc2d88d5988d1ba12ec1 into eb6e2b318715456887e80c5eb9750321(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:30,664 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:30,664 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/C, priority=13, startTime=1731372869810; duration=0sec 2024-11-12T00:54:30,664 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:30,664 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:C 2024-11-12T00:54:30,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-12T00:54:30,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:30,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:30,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:30,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:30,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:30,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372930805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372930805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372930806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372930816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372930816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:30,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-12T00:54:30,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:30,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:30,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:30,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:30,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T00:54:30,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/aca884b3b04c495abd5f5779726cef14 2024-11-12T00:54:30,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-12T00:54:30,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:30,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:30,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/1d96ff2b4f854eafba996a0c8d6754c8 is 50, key is test_row_0/C:col10/1731372868985/Put/seqid=0 2024-11-12T00:54:30,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:30,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:30,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:30,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742195_1371 (size=12001) 2024-11-12T00:54:31,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-12T00:54:31,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:31,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:31,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:31,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:31,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:31,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-12T00:54:31,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:31,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:31,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:31,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:31,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:31,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372931315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:31,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:31,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:31,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372931316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:31,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372931316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:31,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:31,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:31,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:31,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372931325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:31,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:31,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:31,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:31,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:31,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372931325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:31,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:31,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:31,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T00:54:31,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/1d96ff2b4f854eafba996a0c8d6754c8 2024-11-12T00:54:31,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/e54bdb845a424587b5d58739b84e73d8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/e54bdb845a424587b5d58739b84e73d8 2024-11-12T00:54:31,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/e54bdb845a424587b5d58739b84e73d8, entries=200, sequenceid=69, filesize=38.6 K 2024-11-12T00:54:31,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/aca884b3b04c495abd5f5779726cef14 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/aca884b3b04c495abd5f5779726cef14 2024-11-12T00:54:31,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/aca884b3b04c495abd5f5779726cef14, entries=150, sequenceid=69, filesize=11.7 K 2024-11-12T00:54:31,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/1d96ff2b4f854eafba996a0c8d6754c8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/1d96ff2b4f854eafba996a0c8d6754c8 2024-11-12T00:54:31,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/1d96ff2b4f854eafba996a0c8d6754c8, entries=150, sequenceid=69, filesize=11.7 K 2024-11-12T00:54:31,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 58142a01560cfc2d88d5988d1ba12ec1 in 1252ms, sequenceid=69, compaction requested=false 2024-11-12T00:54:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:31,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-12T00:54:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:31,440 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T00:54:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:31,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ecd61f8d743942f3a6f52fb83103517f_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_0/A:col10/1731372870170/Put/seqid=0 2024-11-12T00:54:31,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742196_1372 (size=12154) 2024-11-12T00:54:31,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:31,860 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ecd61f8d743942f3a6f52fb83103517f_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ecd61f8d743942f3a6f52fb83103517f_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:31,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/49997e22b91443a691e9a7599ff8ba71, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:31,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/49997e22b91443a691e9a7599ff8ba71 is 175, key is test_row_0/A:col10/1731372870170/Put/seqid=0 2024-11-12T00:54:31,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742197_1373 (size=30955) 2024-11-12T00:54:32,265 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/49997e22b91443a691e9a7599ff8ba71 2024-11-12T00:54:32,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/ad62dd99c5b24c05996d6dfe78ba31dd is 50, key is test_row_0/B:col10/1731372870170/Put/seqid=0 2024-11-12T00:54:32,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742198_1374 (size=12001) 2024-11-12T00:54:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:32,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:32,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T00:54:32,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372932360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:32,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372932362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:32,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:32,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372932367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:32,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372932368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:32,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372932370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:32,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372932475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:32,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372932475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:32,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372932486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372932486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372932486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:32,679 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/ad62dd99c5b24c05996d6dfe78ba31dd 2024-11-12T00:54:32,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/73a4647d282a4ec0a628a3695e629e9f is 50, key is test_row_0/C:col10/1731372870170/Put/seqid=0 2024-11-12T00:54:32,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742199_1375 (size=12001) 2024-11-12T00:54:32,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372932685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:32,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372932696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:32,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372932705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:32,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372932705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:32,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:32,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372932705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:32,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372932995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372933015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372933016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372933016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372933016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,091 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/73a4647d282a4ec0a628a3695e629e9f 2024-11-12T00:54:33,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/49997e22b91443a691e9a7599ff8ba71 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/49997e22b91443a691e9a7599ff8ba71 2024-11-12T00:54:33,098 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/49997e22b91443a691e9a7599ff8ba71, entries=150, sequenceid=85, filesize=30.2 K 2024-11-12T00:54:33,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/ad62dd99c5b24c05996d6dfe78ba31dd as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ad62dd99c5b24c05996d6dfe78ba31dd 2024-11-12T00:54:33,101 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ad62dd99c5b24c05996d6dfe78ba31dd, entries=150, sequenceid=85, filesize=11.7 K 2024-11-12T00:54:33,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/73a4647d282a4ec0a628a3695e629e9f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/73a4647d282a4ec0a628a3695e629e9f 2024-11-12T00:54:33,105 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/73a4647d282a4ec0a628a3695e629e9f, entries=150, sequenceid=85, filesize=11.7 K 2024-11-12T00:54:33,106 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 58142a01560cfc2d88d5988d1ba12ec1 in 1666ms, sequenceid=85, compaction requested=true 2024-11-12T00:54:33,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:33,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:33,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-12T00:54:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-12T00:54:33,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-12T00:54:33,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8880 sec 2024-11-12T00:54:33,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.8930 sec 2024-11-12T00:54:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:33,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T00:54:33,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:33,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:33,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:33,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:33,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:33,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:33,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b4f0f5f252e342d090cbe4dc6f3790de_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_0/A:col10/1731372872367/Put/seqid=0 2024-11-12T00:54:33,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742200_1376 (size=14594) 2024-11-12T00:54:33,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372933533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372933534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372933535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372933536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372933541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372933645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372933655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372933655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372933656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372933656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372933855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372933865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372933865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372933866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372933866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:33,921 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:33,924 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b4f0f5f252e342d090cbe4dc6f3790de_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b4f0f5f252e342d090cbe4dc6f3790de_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:33,925 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/7e6ea54c6e114613b96727a0ded93cd8, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:33,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/7e6ea54c6e114613b96727a0ded93cd8 is 175, key is test_row_0/A:col10/1731372872367/Put/seqid=0 2024-11-12T00:54:33,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742201_1377 (size=39549) 2024-11-12T00:54:34,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:34,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372934175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:34,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372934175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372934175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372934175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:34,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:34,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372934186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:34,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:34,330 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/7e6ea54c6e114613b96727a0ded93cd8 2024-11-12T00:54:34,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/5ec7db1bc7fc4668b4ea24802e9ade7b is 50, key is test_row_0/B:col10/1731372872367/Put/seqid=0 2024-11-12T00:54:34,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742202_1378 (size=12001) 2024-11-12T00:54:34,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T00:54:34,385 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:34,386 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:54:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-12T00:54:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T00:54:34,387 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:54:34,388 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:54:34,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:54:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T00:54:34,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T00:54:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:34,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:34,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T00:54:34,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:34,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:34,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:34,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:34,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:34,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372934686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:34,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372934686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372934686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:34,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372934695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T00:54:34,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372934705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:34,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/5ec7db1bc7fc4668b4ea24802e9ade7b 2024-11-12T00:54:34,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/82860a4baea24f56976702449a6c23d9 is 50, key is test_row_0/C:col10/1731372872367/Put/seqid=0 2024-11-12T00:54:34,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742203_1379 (size=12001) 2024-11-12T00:54:34,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T00:54:34,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:34,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:34,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:34,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:34,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:34,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T00:54:34,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:34,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:34,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:34,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:34,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T00:54:35,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/82860a4baea24f56976702449a6c23d9 2024-11-12T00:54:35,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T00:54:35,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:35,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:35,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:35,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:35,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:35,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/7e6ea54c6e114613b96727a0ded93cd8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/7e6ea54c6e114613b96727a0ded93cd8 2024-11-12T00:54:35,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/7e6ea54c6e114613b96727a0ded93cd8, entries=200, sequenceid=105, filesize=38.6 K 2024-11-12T00:54:35,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/5ec7db1bc7fc4668b4ea24802e9ade7b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/5ec7db1bc7fc4668b4ea24802e9ade7b 2024-11-12T00:54:35,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/5ec7db1bc7fc4668b4ea24802e9ade7b, entries=150, sequenceid=105, filesize=11.7 K 2024-11-12T00:54:35,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/82860a4baea24f56976702449a6c23d9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/82860a4baea24f56976702449a6c23d9 2024-11-12T00:54:35,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/82860a4baea24f56976702449a6c23d9, entries=150, sequenceid=105, filesize=11.7 K 2024-11-12T00:54:35,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 58142a01560cfc2d88d5988d1ba12ec1 in 1656ms, sequenceid=105, compaction requested=true 2024-11-12T00:54:35,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:35,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:54:35,169 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:54:35,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:35,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:54:35,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:35,169 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:54:35,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:54:35,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:35,170 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:54:35,170 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:54:35,170 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/A is initiating minor compaction (all files) 2024-11-12T00:54:35,170 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/B is initiating minor compaction (all files) 2024-11-12T00:54:35,170 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/A in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:35,170 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/B in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:35,170 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0b6501314ceb4b3a8a836a61fa0067f7, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/e54bdb845a424587b5d58739b84e73d8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/49997e22b91443a691e9a7599ff8ba71, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/7e6ea54c6e114613b96727a0ded93cd8] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=137.8 K 2024-11-12T00:54:35,170 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ae4f3f167a134a33b91cce0dd14c2213, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/aca884b3b04c495abd5f5779726cef14, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ad62dd99c5b24c05996d6dfe78ba31dd, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/5ec7db1bc7fc4668b4ea24802e9ade7b] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=47.0 K 2024-11-12T00:54:35,170 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:35,170 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0b6501314ceb4b3a8a836a61fa0067f7, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/e54bdb845a424587b5d58739b84e73d8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/49997e22b91443a691e9a7599ff8ba71, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/7e6ea54c6e114613b96727a0ded93cd8] 2024-11-12T00:54:35,170 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting ae4f3f167a134a33b91cce0dd14c2213, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731372868295 2024-11-12T00:54:35,170 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b6501314ceb4b3a8a836a61fa0067f7, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731372868295 2024-11-12T00:54:35,171 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting aca884b3b04c495abd5f5779726cef14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731372868985 2024-11-12T00:54:35,171 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting e54bdb845a424587b5d58739b84e73d8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731372868985 2024-11-12T00:54:35,171 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting ad62dd99c5b24c05996d6dfe78ba31dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731372870170 2024-11-12T00:54:35,171 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49997e22b91443a691e9a7599ff8ba71, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731372870170 2024-11-12T00:54:35,171 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ec7db1bc7fc4668b4ea24802e9ade7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731372872357 2024-11-12T00:54:35,171 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e6ea54c6e114613b96727a0ded93cd8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731372872357 2024-11-12T00:54:35,177 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:35,179 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112376dde07f9ad423db537b6688fe2d122_58142a01560cfc2d88d5988d1ba12ec1 store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:35,179 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#B#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:35,180 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/1ec981a5d8804ed1a931e9e7a4a69b9c is 50, key is test_row_0/B:col10/1731372872367/Put/seqid=0 2024-11-12T00:54:35,181 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112376dde07f9ad423db537b6688fe2d122_58142a01560cfc2d88d5988d1ba12ec1, store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:35,182 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112376dde07f9ad423db537b6688fe2d122_58142a01560cfc2d88d5988d1ba12ec1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:35,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742204_1380 (size=12241) 2024-11-12T00:54:35,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742205_1381 (size=4469) 2024-11-12T00:54:35,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T00:54:35,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:35,304 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T00:54:35,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:35,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:35,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:35,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:35,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:35,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:35,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b3d023b7c8634050ba385464054d61b0_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_0/A:col10/1731372873532/Put/seqid=0 2024-11-12T00:54:35,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742206_1382 (size=12154) 2024-11-12T00:54:35,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:35,319 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b3d023b7c8634050ba385464054d61b0_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b3d023b7c8634050ba385464054d61b0_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:35,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/90ad4bc90bd9482eab51ebbafb0abc3c, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:35,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/90ad4bc90bd9482eab51ebbafb0abc3c is 175, key is test_row_0/A:col10/1731372873532/Put/seqid=0 2024-11-12T00:54:35,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742207_1383 (size=30955) 2024-11-12T00:54:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T00:54:35,588 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/1ec981a5d8804ed1a931e9e7a4a69b9c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/1ec981a5d8804ed1a931e9e7a4a69b9c 2024-11-12T00:54:35,588 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#A#compaction#325 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:35,589 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/78953911d029474d8a0a1922321b1c9b is 175, key is test_row_0/A:col10/1731372872367/Put/seqid=0 2024-11-12T00:54:35,593 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/B of 58142a01560cfc2d88d5988d1ba12ec1 into 1ec981a5d8804ed1a931e9e7a4a69b9c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:35,593 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:35,593 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/B, priority=12, startTime=1731372875169; duration=0sec 2024-11-12T00:54:35,593 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:35,593 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:B 2024-11-12T00:54:35,593 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:54:35,594 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:54:35,595 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/C is initiating minor compaction (all files) 2024-11-12T00:54:35,595 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/C in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:35,595 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/eb6e2b318715456887e80c5eb9750321, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/1d96ff2b4f854eafba996a0c8d6754c8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/73a4647d282a4ec0a628a3695e629e9f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/82860a4baea24f56976702449a6c23d9] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=47.0 K 2024-11-12T00:54:35,595 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting eb6e2b318715456887e80c5eb9750321, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731372868295 2024-11-12T00:54:35,595 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d96ff2b4f854eafba996a0c8d6754c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731372868985 2024-11-12T00:54:35,595 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 73a4647d282a4ec0a628a3695e629e9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731372870170 2024-11-12T00:54:35,596 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 82860a4baea24f56976702449a6c23d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731372872357 2024-11-12T00:54:35,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742208_1384 (size=31195) 2024-11-12T00:54:35,603 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/78953911d029474d8a0a1922321b1c9b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/78953911d029474d8a0a1922321b1c9b 2024-11-12T00:54:35,605 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:35,605 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/acb01ecad3974545bb7d470686d0be5a is 50, key is test_row_0/C:col10/1731372872367/Put/seqid=0 2024-11-12T00:54:35,608 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/A of 58142a01560cfc2d88d5988d1ba12ec1 into 78953911d029474d8a0a1922321b1c9b(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:35,608 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:35,608 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/A, priority=12, startTime=1731372875169; duration=0sec 2024-11-12T00:54:35,608 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:35,608 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:A 2024-11-12T00:54:35,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742209_1385 (size=12241) 2024-11-12T00:54:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:35,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:35,736 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/90ad4bc90bd9482eab51ebbafb0abc3c 2024-11-12T00:54:35,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/7a282e6f9eb148528962d2dd1f73225e is 50, key is test_row_0/B:col10/1731372873532/Put/seqid=0 2024-11-12T00:54:35,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742210_1386 (size=12001) 2024-11-12T00:54:35,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372935740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:35,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731372875732, max timestamp=1731372875750Num uniques 859; 2024-11-12T00:54:35,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T00:54:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:35,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731372875732, max timestamp=1731372875750Num uniques 859; 2024-11-12T00:54:35,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T00:54:35,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731372875732, max timestamp=1731372875750Num uniques 859; 2024-11-12T00:54:35,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T00:54:35,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372935788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:35,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372935789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372935789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:35,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372935790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:35,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:35,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372935856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:35,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372935905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:35,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372935905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:35,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372935906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:35,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372935916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:36,019 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/acb01ecad3974545bb7d470686d0be5a as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/acb01ecad3974545bb7d470686d0be5a 2024-11-12T00:54:36,022 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/C of 58142a01560cfc2d88d5988d1ba12ec1 into acb01ecad3974545bb7d470686d0be5a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:36,022 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:36,022 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/C, priority=12, startTime=1731372875169; duration=0sec 2024-11-12T00:54:36,022 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:36,022 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:C 2024-11-12T00:54:36,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372936065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:36,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372936125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372936126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:36,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372936126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372936126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:36,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:36,146 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/7a282e6f9eb148528962d2dd1f73225e 2024-11-12T00:54:36,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/391d5e9adf5e48b08d85a356d0947625 is 50, key is test_row_0/C:col10/1731372873532/Put/seqid=0 2024-11-12T00:54:36,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742211_1387 (size=12001) 2024-11-12T00:54:36,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372936376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:36,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372936445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:36,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372936445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372936445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372936445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T00:54:36,562 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/391d5e9adf5e48b08d85a356d0947625 2024-11-12T00:54:36,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/90ad4bc90bd9482eab51ebbafb0abc3c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/90ad4bc90bd9482eab51ebbafb0abc3c 2024-11-12T00:54:36,568 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/90ad4bc90bd9482eab51ebbafb0abc3c, entries=150, sequenceid=117, filesize=30.2 K 2024-11-12T00:54:36,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/7a282e6f9eb148528962d2dd1f73225e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/7a282e6f9eb148528962d2dd1f73225e 2024-11-12T00:54:36,571 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/7a282e6f9eb148528962d2dd1f73225e, entries=150, sequenceid=117, filesize=11.7 K 2024-11-12T00:54:36,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/391d5e9adf5e48b08d85a356d0947625 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/391d5e9adf5e48b08d85a356d0947625 2024-11-12T00:54:36,574 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/391d5e9adf5e48b08d85a356d0947625, entries=150, sequenceid=117, filesize=11.7 K 2024-11-12T00:54:36,575 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 58142a01560cfc2d88d5988d1ba12ec1 in 1271ms, sequenceid=117, compaction requested=false 2024-11-12T00:54:36,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:36,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:36,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-12T00:54:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-12T00:54:36,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-12T00:54:36,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1880 sec 2024-11-12T00:54:36,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.1920 sec 2024-11-12T00:54:36,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:36,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-12T00:54:36,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:36,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:36,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:36,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:36,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:36,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:36,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112777fc906ee314b989cab93b3b9e96df4_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_0/A:col10/1731372875789/Put/seqid=0 2024-11-12T00:54:36,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742212_1388 (size=14794) 2024-11-12T00:54:36,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:36,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372936965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:36,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:36,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372936966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:36,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:36,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372936966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:36,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372936966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:36,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372936966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:37,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372937075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:37,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:37,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:37,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:37,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372937285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:37,316 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:37,319 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112777fc906ee314b989cab93b3b9e96df4_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112777fc906ee314b989cab93b3b9e96df4_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:37,320 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/5d086e15ea9b4218b8bf18276b8b91e5, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:37,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/5d086e15ea9b4218b8bf18276b8b91e5 is 175, key is test_row_0/A:col10/1731372875789/Put/seqid=0 2024-11-12T00:54:37,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742213_1389 (size=39749) 2024-11-12T00:54:37,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372937606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:37,724 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/5d086e15ea9b4218b8bf18276b8b91e5 2024-11-12T00:54:37,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/8718ccadf094469c90eee9ccc7f4421c is 50, key is test_row_0/B:col10/1731372875789/Put/seqid=0 2024-11-12T00:54:37,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742214_1390 (size=12151) 2024-11-12T00:54:37,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372937975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:37,986 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 58142a01560cfc2d88d5988d1ba12ec1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:37,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:37,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:37,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:37,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372937985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:37,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372937985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:37,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372937986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:37,993 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_1' in 58142a01560cfc2d88d5988d1ba12ec1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:37,993 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 58142a01560cfc2d88d5988d1ba12ec1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:37,993 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 58142a01560cfc2d88d5988d1ba12ec1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:38,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/8718ccadf094469c90eee9ccc7f4421c 2024-11-12T00:54:38,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372938126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:38,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/f3550b08dab44e23a0b4ae6f17ba9633 is 50, key is test_row_0/C:col10/1731372875789/Put/seqid=0 2024-11-12T00:54:38,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742215_1391 (size=12151) 2024-11-12T00:54:38,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/f3550b08dab44e23a0b4ae6f17ba9633 2024-11-12T00:54:38,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/5d086e15ea9b4218b8bf18276b8b91e5 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/5d086e15ea9b4218b8bf18276b8b91e5 2024-11-12T00:54:38,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/5d086e15ea9b4218b8bf18276b8b91e5, entries=200, sequenceid=154, filesize=38.8 K 2024-11-12T00:54:38,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/8718ccadf094469c90eee9ccc7f4421c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/8718ccadf094469c90eee9ccc7f4421c 2024-11-12T00:54:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T00:54:38,556 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:38,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:54:38,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/8718ccadf094469c90eee9ccc7f4421c, entries=150, sequenceid=154, filesize=11.9 K 2024-11-12T00:54:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-12T00:54:38,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/f3550b08dab44e23a0b4ae6f17ba9633 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/f3550b08dab44e23a0b4ae6f17ba9633 2024-11-12T00:54:38,558 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:54:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T00:54:38,558 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:54:38,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:54:38,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/f3550b08dab44e23a0b4ae6f17ba9633, entries=150, sequenceid=154, filesize=11.9 K 2024-11-12T00:54:38,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 58142a01560cfc2d88d5988d1ba12ec1 in 1664ms, sequenceid=154, compaction requested=true 2024-11-12T00:54:38,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:38,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:54:38,564 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:38,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:38,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:54:38,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:38,564 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:38,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:54:38,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:38,565 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:38,565 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:38,565 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/A is initiating minor compaction (all files) 2024-11-12T00:54:38,565 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/B is initiating minor compaction (all files) 2024-11-12T00:54:38,565 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/A in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:38,565 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/B in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:38,565 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/78953911d029474d8a0a1922321b1c9b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/90ad4bc90bd9482eab51ebbafb0abc3c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/5d086e15ea9b4218b8bf18276b8b91e5] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=99.5 K 2024-11-12T00:54:38,565 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/1ec981a5d8804ed1a931e9e7a4a69b9c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/7a282e6f9eb148528962d2dd1f73225e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/8718ccadf094469c90eee9ccc7f4421c] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=35.5 K 2024-11-12T00:54:38,565 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:38,565 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/78953911d029474d8a0a1922321b1c9b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/90ad4bc90bd9482eab51ebbafb0abc3c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/5d086e15ea9b4218b8bf18276b8b91e5] 2024-11-12T00:54:38,566 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ec981a5d8804ed1a931e9e7a4a69b9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731372872357 2024-11-12T00:54:38,566 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78953911d029474d8a0a1922321b1c9b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731372872357 2024-11-12T00:54:38,566 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a282e6f9eb148528962d2dd1f73225e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731372873532 2024-11-12T00:54:38,566 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90ad4bc90bd9482eab51ebbafb0abc3c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731372873532 2024-11-12T00:54:38,566 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 8718ccadf094469c90eee9ccc7f4421c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731372875787 2024-11-12T00:54:38,566 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d086e15ea9b4218b8bf18276b8b91e5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731372875787 2024-11-12T00:54:38,571 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:38,573 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#B#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:38,573 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/6d4cf723ca254277a6089da03fb9efd0 is 50, key is test_row_0/B:col10/1731372875789/Put/seqid=0 2024-11-12T00:54:38,577 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411125cdab9291d0c40debfb0c3bba75d7c91_58142a01560cfc2d88d5988d1ba12ec1 store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:38,580 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411125cdab9291d0c40debfb0c3bba75d7c91_58142a01560cfc2d88d5988d1ba12ec1, store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:38,580 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125cdab9291d0c40debfb0c3bba75d7c91_58142a01560cfc2d88d5988d1ba12ec1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:38,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742216_1392 (size=12493) 2024-11-12T00:54:38,586 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/6d4cf723ca254277a6089da03fb9efd0 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/6d4cf723ca254277a6089da03fb9efd0 2024-11-12T00:54:38,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742217_1393 (size=4469) 2024-11-12T00:54:38,591 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#A#compaction#334 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:38,592 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/62dbf287121943f78af7cbcaf4ddd1e6 is 175, key is test_row_0/A:col10/1731372875789/Put/seqid=0 2024-11-12T00:54:38,592 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/B of 58142a01560cfc2d88d5988d1ba12ec1 into 6d4cf723ca254277a6089da03fb9efd0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:38,592 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:38,592 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/B, priority=13, startTime=1731372878564; duration=0sec 2024-11-12T00:54:38,592 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:38,592 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:B 2024-11-12T00:54:38,592 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:38,593 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:38,593 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/C is initiating minor compaction (all files) 2024-11-12T00:54:38,593 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/C in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:38,593 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/acb01ecad3974545bb7d470686d0be5a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/391d5e9adf5e48b08d85a356d0947625, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/f3550b08dab44e23a0b4ae6f17ba9633] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=35.5 K 2024-11-12T00:54:38,594 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting acb01ecad3974545bb7d470686d0be5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731372872357 2024-11-12T00:54:38,594 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 391d5e9adf5e48b08d85a356d0947625, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731372873532 2024-11-12T00:54:38,594 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting f3550b08dab44e23a0b4ae6f17ba9633, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731372875787 2024-11-12T00:54:38,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742218_1394 (size=31447) 2024-11-12T00:54:38,599 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/62dbf287121943f78af7cbcaf4ddd1e6 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/62dbf287121943f78af7cbcaf4ddd1e6 2024-11-12T00:54:38,601 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#C#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:38,602 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/2bef979dedc848dd97730e84f42f1120 is 50, key is test_row_0/C:col10/1731372875789/Put/seqid=0 2024-11-12T00:54:38,603 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/A of 58142a01560cfc2d88d5988d1ba12ec1 into 62dbf287121943f78af7cbcaf4ddd1e6(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:38,603 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:38,603 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/A, priority=13, startTime=1731372878564; duration=0sec 2024-11-12T00:54:38,603 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:38,603 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:A 2024-11-12T00:54:38,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742219_1395 (size=12493) 2024-11-12T00:54:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T00:54:38,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T00:54:38,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:38,711 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T00:54:38,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:38,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:38,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:38,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:38,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:38,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:38,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125a02da066bea4841a234176769ce31ed_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_0/A:col10/1731372876942/Put/seqid=0 2024-11-12T00:54:38,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742220_1396 (size=12304) 2024-11-12T00:54:38,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T00:54:39,017 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/2bef979dedc848dd97730e84f42f1120 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/2bef979dedc848dd97730e84f42f1120 2024-11-12T00:54:39,021 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/C of 58142a01560cfc2d88d5988d1ba12ec1 into 2bef979dedc848dd97730e84f42f1120(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:39,021 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:39,021 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/C, priority=13, startTime=1731372878564; duration=0sec 2024-11-12T00:54:39,021 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:39,021 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:C 2024-11-12T00:54:39,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:39,124 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125a02da066bea4841a234176769ce31ed_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125a02da066bea4841a234176769ce31ed_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:39,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/ecd4561d24b9483ca7656f767f7e1bb5, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:39,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/ecd4561d24b9483ca7656f767f7e1bb5 is 175, key is test_row_0/A:col10/1731372876942/Put/seqid=0 2024-11-12T00:54:39,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742221_1397 (size=31105) 2024-11-12T00:54:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:39,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T00:54:39,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372879205, max timestamp=1731372879247Num uniques 859; 2024-11-12T00:54:39,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372879205, max timestamp=1731372879247Num uniques 859; 2024-11-12T00:54:39,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:54:39,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:54:39,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372879205, max timestamp=1731372879247Num uniques 859; 2024-11-12T00:54:39,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:54:39,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372939356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:39,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:39,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:39,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:39,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372939465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:39,529 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/ecd4561d24b9483ca7656f767f7e1bb5 2024-11-12T00:54:39,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/960cfa1e199f45cfbc8b5b846611c49b is 50, key is test_row_0/B:col10/1731372876942/Put/seqid=0 2024-11-12T00:54:39,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742222_1398 (size=12151) 2024-11-12T00:54:39,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372939676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T00:54:39,938 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/960cfa1e199f45cfbc8b5b846611c49b 2024-11-12T00:54:39,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/ee2705072f134dd1b3baa3d58dc97638 is 50, key is test_row_0/C:col10/1731372876942/Put/seqid=0 2024-11-12T00:54:39,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742223_1399 (size=12151) 2024-11-12T00:54:40,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372939995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:40,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372940006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:40,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372940006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:40,016 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_1' in 58142a01560cfc2d88d5988d1ba12ec1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:40,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372940006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:40,017 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 58142a01560cfc2d88d5988d1ba12ec1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:40,017 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 58142a01560cfc2d88d5988d1ba12ec1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:40,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372940015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:40,019 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 58142a01560cfc2d88d5988d1ba12ec1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:40,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:40,349 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/ee2705072f134dd1b3baa3d58dc97638 2024-11-12T00:54:40,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/ecd4561d24b9483ca7656f767f7e1bb5 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/ecd4561d24b9483ca7656f767f7e1bb5 2024-11-12T00:54:40,355 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/ecd4561d24b9483ca7656f767f7e1bb5, entries=150, sequenceid=167, filesize=30.4 K 2024-11-12T00:54:40,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/960cfa1e199f45cfbc8b5b846611c49b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/960cfa1e199f45cfbc8b5b846611c49b 2024-11-12T00:54:40,359 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/960cfa1e199f45cfbc8b5b846611c49b, entries=150, sequenceid=167, filesize=11.9 K 2024-11-12T00:54:40,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/ee2705072f134dd1b3baa3d58dc97638 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/ee2705072f134dd1b3baa3d58dc97638 2024-11-12T00:54:40,362 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/ee2705072f134dd1b3baa3d58dc97638, entries=150, sequenceid=167, filesize=11.9 K 2024-11-12T00:54:40,363 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 58142a01560cfc2d88d5988d1ba12ec1 in 1652ms, sequenceid=167, compaction requested=false 2024-11-12T00:54:40,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:40,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:40,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-12T00:54:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-12T00:54:40,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-12T00:54:40,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8050 sec 2024-11-12T00:54:40,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.8100 sec 2024-11-12T00:54:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:40,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-12T00:54:40,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:40,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:40,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:40,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:40,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:40,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:40,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c2bca635964e46b9b2ab773ffd81a8e6_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_0/A:col10/1731372879331/Put/seqid=0 2024-11-12T00:54:40,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742224_1400 (size=14794) 2024-11-12T00:54:40,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372940562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:40,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372940675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T00:54:40,706 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:40,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:54:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-12T00:54:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T00:54:40,707 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:54:40,708 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:54:40,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:54:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T00:54:40,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T00:54:40,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:40,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:40,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:40,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:40,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:40,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372940885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:40,933 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:40,936 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c2bca635964e46b9b2ab773ffd81a8e6_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c2bca635964e46b9b2ab773ffd81a8e6_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:40,937 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/de96ba41ff784432a418a2c933e38ff2, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:40,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/de96ba41ff784432a418a2c933e38ff2 is 175, key is test_row_0/A:col10/1731372879331/Put/seqid=0 2024-11-12T00:54:40,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742225_1401 (size=39749) 2024-11-12T00:54:40,942 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/de96ba41ff784432a418a2c933e38ff2 2024-11-12T00:54:40,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/dfbfcbaf8a3246d98a89c5ab0f1cc131 is 50, key is test_row_0/B:col10/1731372879331/Put/seqid=0 2024-11-12T00:54:40,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742226_1402 (size=12151) 2024-11-12T00:54:40,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/dfbfcbaf8a3246d98a89c5ab0f1cc131 2024-11-12T00:54:40,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/4ea6633b642748dba6ab235ce94d8366 is 50, key is test_row_0/C:col10/1731372879331/Put/seqid=0 2024-11-12T00:54:40,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742227_1403 (size=12151) 2024-11-12T00:54:40,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/4ea6633b642748dba6ab235ce94d8366 2024-11-12T00:54:40,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/de96ba41ff784432a418a2c933e38ff2 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/de96ba41ff784432a418a2c933e38ff2 2024-11-12T00:54:40,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/de96ba41ff784432a418a2c933e38ff2, entries=200, sequenceid=205, filesize=38.8 K 2024-11-12T00:54:40,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/dfbfcbaf8a3246d98a89c5ab0f1cc131 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/dfbfcbaf8a3246d98a89c5ab0f1cc131 2024-11-12T00:54:40,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/dfbfcbaf8a3246d98a89c5ab0f1cc131, entries=150, sequenceid=205, filesize=11.9 K 2024-11-12T00:54:40,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/4ea6633b642748dba6ab235ce94d8366 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/4ea6633b642748dba6ab235ce94d8366 2024-11-12T00:54:40,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/4ea6633b642748dba6ab235ce94d8366, entries=150, sequenceid=205, filesize=11.9 K 2024-11-12T00:54:40,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 58142a01560cfc2d88d5988d1ba12ec1 in 463ms, sequenceid=205, compaction requested=true 2024-11-12T00:54:40,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:40,982 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:40,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:54:40,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:40,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:54:40,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:40,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:54:40,982 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:40,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:40,985 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:40,985 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:40,985 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/B is initiating minor compaction (all files) 2024-11-12T00:54:40,985 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/A is initiating minor compaction (all files) 2024-11-12T00:54:40,985 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/A in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:40,985 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/B in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:40,985 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/62dbf287121943f78af7cbcaf4ddd1e6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/ecd4561d24b9483ca7656f767f7e1bb5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/de96ba41ff784432a418a2c933e38ff2] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=99.9 K 2024-11-12T00:54:40,985 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/6d4cf723ca254277a6089da03fb9efd0, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/960cfa1e199f45cfbc8b5b846611c49b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/dfbfcbaf8a3246d98a89c5ab0f1cc131] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=35.9 K 2024-11-12T00:54:40,985 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:40,985 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/62dbf287121943f78af7cbcaf4ddd1e6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/ecd4561d24b9483ca7656f767f7e1bb5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/de96ba41ff784432a418a2c933e38ff2] 2024-11-12T00:54:40,985 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d4cf723ca254277a6089da03fb9efd0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731372875787 2024-11-12T00:54:40,985 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62dbf287121943f78af7cbcaf4ddd1e6, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731372875787 2024-11-12T00:54:40,986 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 960cfa1e199f45cfbc8b5b846611c49b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731372876942 2024-11-12T00:54:40,986 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecd4561d24b9483ca7656f767f7e1bb5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731372876942 2024-11-12T00:54:40,986 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting dfbfcbaf8a3246d98a89c5ab0f1cc131, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731372879331 2024-11-12T00:54:40,986 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting de96ba41ff784432a418a2c933e38ff2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731372879331 2024-11-12T00:54:40,991 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:40,993 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112a49139c9756e410c98f3930fe01862df_58142a01560cfc2d88d5988d1ba12ec1 store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:40,995 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112a49139c9756e410c98f3930fe01862df_58142a01560cfc2d88d5988d1ba12ec1, store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:40,995 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a49139c9756e410c98f3930fe01862df_58142a01560cfc2d88d5988d1ba12ec1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:40,997 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#B#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:40,998 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/10a96256877b414e97e050c3a3e51b7b is 50, key is test_row_0/B:col10/1731372879331/Put/seqid=0 2024-11-12T00:54:41,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742229_1405 (size=12595) 2024-11-12T00:54:41,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742228_1404 (size=4469) 2024-11-12T00:54:41,010 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/10a96256877b414e97e050c3a3e51b7b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/10a96256877b414e97e050c3a3e51b7b 2024-11-12T00:54:41,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T00:54:41,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:41,013 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T00:54:41,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:41,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:41,014 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/B of 58142a01560cfc2d88d5988d1ba12ec1 into 10a96256877b414e97e050c3a3e51b7b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:41,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:41,014 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:41,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:41,014 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/B, priority=13, startTime=1731372880982; duration=0sec 2024-11-12T00:54:41,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:41,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:41,014 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:41,014 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:B 2024-11-12T00:54:41,014 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:41,015 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:41,015 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/C is initiating minor compaction (all files) 2024-11-12T00:54:41,015 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/C in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:41,015 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/2bef979dedc848dd97730e84f42f1120, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/ee2705072f134dd1b3baa3d58dc97638, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/4ea6633b642748dba6ab235ce94d8366] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=35.9 K 2024-11-12T00:54:41,015 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bef979dedc848dd97730e84f42f1120, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731372875787 2024-11-12T00:54:41,016 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting ee2705072f134dd1b3baa3d58dc97638, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731372876942 2024-11-12T00:54:41,016 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ea6633b642748dba6ab235ce94d8366, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731372879331 2024-11-12T00:54:41,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111248b721fc41e746fda9676738da274d8f_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_0/A:col10/1731372880549/Put/seqid=0 2024-11-12T00:54:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T00:54:41,026 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#C#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:41,026 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/379ae6d0e5174a4890cd6bd22dbb8982 is 50, key is test_row_0/C:col10/1731372879331/Put/seqid=0 2024-11-12T00:54:41,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742230_1406 (size=12304) 2024-11-12T00:54:41,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742231_1407 (size=12595) 2024-11-12T00:54:41,043 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/379ae6d0e5174a4890cd6bd22dbb8982 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/379ae6d0e5174a4890cd6bd22dbb8982 2024-11-12T00:54:41,047 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/C of 58142a01560cfc2d88d5988d1ba12ec1 into 379ae6d0e5174a4890cd6bd22dbb8982(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:41,047 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:41,047 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/C, priority=13, startTime=1731372880982; duration=0sec 2024-11-12T00:54:41,048 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:41,048 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:C 2024-11-12T00:54:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:41,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:41,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731372881264, max timestamp=1731372881305Num uniques 859; 2024-11-12T00:54:41,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:54:41,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731372881264, max timestamp=1731372881305Num uniques 859; 2024-11-12T00:54:41,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731372881264, max timestamp=1731372881305Num uniques 859; 2024-11-12T00:54:41,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:54:41,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:54:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T00:54:41,405 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#A#compaction#342 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:41,406 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/cbb91c2280064e43bf8b5c93b76854c8 is 175, key is test_row_0/A:col10/1731372879331/Put/seqid=0 2024-11-12T00:54:41,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742232_1408 (size=31549) 2024-11-12T00:54:41,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372941405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:41,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:41,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:41,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:41,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:41,435 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111248b721fc41e746fda9676738da274d8f_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111248b721fc41e746fda9676738da274d8f_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:41,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/c654aa6012f24b8a8df40ea78a52f47b, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:41,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/c654aa6012f24b8a8df40ea78a52f47b is 175, key is test_row_0/A:col10/1731372880549/Put/seqid=0 2024-11-12T00:54:41,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742233_1409 (size=31105) 2024-11-12T00:54:41,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372941515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:41,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372941726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:41,813 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/cbb91c2280064e43bf8b5c93b76854c8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/cbb91c2280064e43bf8b5c93b76854c8 2024-11-12T00:54:41,817 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/A of 58142a01560cfc2d88d5988d1ba12ec1 into cbb91c2280064e43bf8b5c93b76854c8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:41,817 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:41,817 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/A, priority=13, startTime=1731372880982; duration=0sec 2024-11-12T00:54:41,817 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:41,817 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:A 2024-11-12T00:54:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T00:54:41,849 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/c654aa6012f24b8a8df40ea78a52f47b 2024-11-12T00:54:41,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/192d964b3d094f2783a20342b76cd5d9 is 50, key is test_row_0/B:col10/1731372880549/Put/seqid=0 2024-11-12T00:54:41,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742234_1410 (size=12151) 2024-11-12T00:54:42,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372942035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:42,266 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/192d964b3d094f2783a20342b76cd5d9 2024-11-12T00:54:42,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/02013c9ea6b04c218f20b4a77ec57692 is 50, key is test_row_0/C:col10/1731372880549/Put/seqid=0 2024-11-12T00:54:42,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742235_1411 (size=12151) 2024-11-12T00:54:42,276 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/02013c9ea6b04c218f20b4a77ec57692 2024-11-12T00:54:42,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/c654aa6012f24b8a8df40ea78a52f47b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/c654aa6012f24b8a8df40ea78a52f47b 2024-11-12T00:54:42,282 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/c654aa6012f24b8a8df40ea78a52f47b, entries=150, sequenceid=216, filesize=30.4 K 2024-11-12T00:54:42,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/192d964b3d094f2783a20342b76cd5d9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/192d964b3d094f2783a20342b76cd5d9 2024-11-12T00:54:42,286 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/192d964b3d094f2783a20342b76cd5d9, entries=150, sequenceid=216, filesize=11.9 K 2024-11-12T00:54:42,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/02013c9ea6b04c218f20b4a77ec57692 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/02013c9ea6b04c218f20b4a77ec57692 2024-11-12T00:54:42,291 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/02013c9ea6b04c218f20b4a77ec57692, entries=150, sequenceid=216, filesize=11.9 K 2024-11-12T00:54:42,291 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 58142a01560cfc2d88d5988d1ba12ec1 in 1278ms, sequenceid=216, compaction requested=false 2024-11-12T00:54:42,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:42,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:42,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-12T00:54:42,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-12T00:54:42,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-12T00:54:42,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5840 sec 2024-11-12T00:54:42,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.5880 sec 2024-11-12T00:54:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:42,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-12T00:54:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:42,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112615a15e8269f4a9db6d06ad8d7668e60_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_0/A:col10/1731372881404/Put/seqid=0 2024-11-12T00:54:42,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742236_1412 (size=14794) 2024-11-12T00:54:42,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372942593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:42,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372942705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T00:54:42,855 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:42,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:54:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-12T00:54:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T00:54:42,857 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:54:42,858 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:54:42,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:54:42,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372942916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:42,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:42,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:42,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:42,960 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:42,962 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112615a15e8269f4a9db6d06ad8d7668e60_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112615a15e8269f4a9db6d06ad8d7668e60_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:42,963 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/91766d195d32496da8da4fbb8cb089db, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:42,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/91766d195d32496da8da4fbb8cb089db is 175, key is test_row_0/A:col10/1731372881404/Put/seqid=0 2024-11-12T00:54:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T00:54:42,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742237_1413 (size=39749) 2024-11-12T00:54:43,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T00:54:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:43,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T00:54:43,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:43,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:43,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:43,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T00:54:43,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372943235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:43,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:43,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:43,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:43,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T00:54:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:43,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,367 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/91766d195d32496da8da4fbb8cb089db 2024-11-12T00:54:43,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/828bdeb88d454e17b751d18a610ef116 is 50, key is test_row_0/B:col10/1731372881404/Put/seqid=0 2024-11-12T00:54:43,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742238_1414 (size=12151) 2024-11-12T00:54:43,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T00:54:43,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:43,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:43,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:43,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T00:54:43,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T00:54:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:43,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372943745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:43,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T00:54:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:43,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/828bdeb88d454e17b751d18a610ef116 2024-11-12T00:54:43,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/2f3e862f33ff4901bdea78f47a0078c8 is 50, key is test_row_0/C:col10/1731372881404/Put/seqid=0 2024-11-12T00:54:43,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742239_1415 (size=12151) 2024-11-12T00:54:43,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T00:54:43,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:43,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:43,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:43,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T00:54:44,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58764 deadline: 1731372944035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:44,038 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 58142a01560cfc2d88d5988d1ba12ec1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:44,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58726 deadline: 1731372944046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:44,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58750 deadline: 1731372944046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:44,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:44,050 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_1' in 58142a01560cfc2d88d5988d1ba12ec1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58792 deadline: 1731372944046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:44,050 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 58142a01560cfc2d88d5988d1ba12ec1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:44,050 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 58142a01560cfc2d88d5988d1ba12ec1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:54:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:44,054 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T00:54:44,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T00:54:44,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:44,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:44,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:44,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:44,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:44,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/2f3e862f33ff4901bdea78f47a0078c8 2024-11-12T00:54:44,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/91766d195d32496da8da4fbb8cb089db as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/91766d195d32496da8da4fbb8cb089db 2024-11-12T00:54:44,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/91766d195d32496da8da4fbb8cb089db, entries=200, sequenceid=255, filesize=38.8 K 2024-11-12T00:54:44,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/828bdeb88d454e17b751d18a610ef116 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/828bdeb88d454e17b751d18a610ef116 2024-11-12T00:54:44,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/828bdeb88d454e17b751d18a610ef116, entries=150, sequenceid=255, filesize=11.9 K 2024-11-12T00:54:44,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/2f3e862f33ff4901bdea78f47a0078c8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/2f3e862f33ff4901bdea78f47a0078c8 2024-11-12T00:54:44,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/2f3e862f33ff4901bdea78f47a0078c8, entries=150, sequenceid=255, filesize=11.9 K 2024-11-12T00:54:44,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 58142a01560cfc2d88d5988d1ba12ec1 in 1653ms, sequenceid=255, compaction requested=true 2024-11-12T00:54:44,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:44,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:54:44,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:44,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:54:44,203 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:44,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:44,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:54:44,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:44,203 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:44,204 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:44,204 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:44,204 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/A is initiating minor compaction (all files) 2024-11-12T00:54:44,204 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/B is initiating minor compaction (all files) 2024-11-12T00:54:44,204 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/A in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:44,204 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/B in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:44,204 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/cbb91c2280064e43bf8b5c93b76854c8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/c654aa6012f24b8a8df40ea78a52f47b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/91766d195d32496da8da4fbb8cb089db] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=100.0 K 2024-11-12T00:54:44,204 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/10a96256877b414e97e050c3a3e51b7b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/192d964b3d094f2783a20342b76cd5d9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/828bdeb88d454e17b751d18a610ef116] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=36.0 K 2024-11-12T00:54:44,204 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:44,204 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/cbb91c2280064e43bf8b5c93b76854c8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/c654aa6012f24b8a8df40ea78a52f47b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/91766d195d32496da8da4fbb8cb089db] 2024-11-12T00:54:44,204 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 10a96256877b414e97e050c3a3e51b7b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731372879331 2024-11-12T00:54:44,204 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbb91c2280064e43bf8b5c93b76854c8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731372879331 2024-11-12T00:54:44,205 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 192d964b3d094f2783a20342b76cd5d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731372880529 2024-11-12T00:54:44,205 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting c654aa6012f24b8a8df40ea78a52f47b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731372880529 2024-11-12T00:54:44,205 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 828bdeb88d454e17b751d18a610ef116, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731372881330 2024-11-12T00:54:44,205 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91766d195d32496da8da4fbb8cb089db, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731372881330 2024-11-12T00:54:44,210 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:44,211 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112e1908be3b2b24517b3e56e14bf44bfd0_58142a01560cfc2d88d5988d1ba12ec1 store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:44,212 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#B#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:44,212 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/84e37447b17440a1b94c34cf9eece696 is 50, key is test_row_0/B:col10/1731372881404/Put/seqid=0 2024-11-12T00:54:44,214 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112e1908be3b2b24517b3e56e14bf44bfd0_58142a01560cfc2d88d5988d1ba12ec1, store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:44,214 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e1908be3b2b24517b3e56e14bf44bfd0_58142a01560cfc2d88d5988d1ba12ec1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:44,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742240_1416 (size=4469) 2024-11-12T00:54:44,224 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#A#compaction#352 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:44,225 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/0052330284ee447c902dfab6beaf3631 is 175, key is test_row_0/A:col10/1731372881404/Put/seqid=0 2024-11-12T00:54:44,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T00:54:44,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:44,231 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-12T00:54:44,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:44,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:44,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:44,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:44,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:44,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:44,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f95e286395784a50aefbdfeab7f665ee_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_0/A:col10/1731372882556/Put/seqid=0 2024-11-12T00:54:44,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742241_1417 (size=12697) 2024-11-12T00:54:44,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742242_1418 (size=31651) 2024-11-12T00:54:44,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742243_1419 (size=12454) 2024-11-12T00:54:44,641 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/84e37447b17440a1b94c34cf9eece696 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/84e37447b17440a1b94c34cf9eece696 2024-11-12T00:54:44,645 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/B of 58142a01560cfc2d88d5988d1ba12ec1 into 84e37447b17440a1b94c34cf9eece696(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:44,645 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:44,645 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/B, priority=13, startTime=1731372884203; duration=0sec 2024-11-12T00:54:44,645 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:44,646 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:B 2024-11-12T00:54:44,646 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/0052330284ee447c902dfab6beaf3631 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0052330284ee447c902dfab6beaf3631 2024-11-12T00:54:44,646 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:44,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:44,646 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:44,646 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/C is initiating minor compaction (all files) 2024-11-12T00:54:44,647 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/C in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:44,647 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/379ae6d0e5174a4890cd6bd22dbb8982, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/02013c9ea6b04c218f20b4a77ec57692, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/2f3e862f33ff4901bdea78f47a0078c8] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=36.0 K 2024-11-12T00:54:44,647 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 379ae6d0e5174a4890cd6bd22dbb8982, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731372879331 2024-11-12T00:54:44,647 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 02013c9ea6b04c218f20b4a77ec57692, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731372880529 2024-11-12T00:54:44,647 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f3e862f33ff4901bdea78f47a0078c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731372881330 2024-11-12T00:54:44,651 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/A of 58142a01560cfc2d88d5988d1ba12ec1 into 0052330284ee447c902dfab6beaf3631(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:44,651 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:44,651 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/A, priority=13, startTime=1731372884203; duration=0sec 2024-11-12T00:54:44,651 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:44,651 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:A 2024-11-12T00:54:44,651 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f95e286395784a50aefbdfeab7f665ee_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f95e286395784a50aefbdfeab7f665ee_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:44,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/0e03f580a05649b68f06f99492970fb9, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:44,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/0e03f580a05649b68f06f99492970fb9 is 175, key is test_row_0/A:col10/1731372882556/Put/seqid=0 2024-11-12T00:54:44,655 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#C#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:44,656 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/95d819fc0ef04bcdb744dc2e95c5b924 is 50, key is test_row_0/C:col10/1731372881404/Put/seqid=0 2024-11-12T00:54:44,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742244_1420 (size=31255) 2024-11-12T00:54:44,660 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/0e03f580a05649b68f06f99492970fb9 2024-11-12T00:54:44,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742245_1421 (size=12697) 2024-11-12T00:54:44,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/100fe395b70f4051a906f64aa0b9d5af is 50, key is test_row_0/B:col10/1731372882556/Put/seqid=0 2024-11-12T00:54:44,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742246_1422 (size=12301) 2024-11-12T00:54:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:44,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. as already flushing 2024-11-12T00:54:44,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372884785, max timestamp=1731372884865Num uniques 859; 2024-11-12T00:54:44,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372884785, max timestamp=1731372884865Num uniques 859; 2024-11-12T00:54:44,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372884785, max timestamp=1731372884865Num uniques 859; 2024-11-12T00:54:44,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:54:44,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:54:44,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:54:44,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731372884932, max timestamp=1731372884994Num uniques 1018; 2024-11-12T00:54:44,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731372884932, max timestamp=1731372884994Num uniques 1018; 2024-11-12T00:54:44,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731372884932, max timestamp=1731372884994Num uniques 1018; 2024-11-12T00:54:44,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T00:54:44,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T00:54:44,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T00:54:45,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T00:54:45,069 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/95d819fc0ef04bcdb744dc2e95c5b924 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/95d819fc0ef04bcdb744dc2e95c5b924 2024-11-12T00:54:45,072 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/C of 58142a01560cfc2d88d5988d1ba12ec1 into 95d819fc0ef04bcdb744dc2e95c5b924(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:45,072 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:45,072 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/C, priority=13, startTime=1731372884203; duration=0sec 2024-11-12T00:54:45,073 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:45,073 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:C 2024-11-12T00:54:45,076 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/100fe395b70f4051a906f64aa0b9d5af 2024-11-12T00:54:45,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/534d596c6a0e422fb5868ccbb6069a50 is 50, key is test_row_0/C:col10/1731372882556/Put/seqid=0 2024-11-12T00:54:45,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742247_1423 (size=12301) 2024-11-12T00:54:45,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731372885077, max timestamp=1731372885124Num uniques 1027; 2024-11-12T00:54:45,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731372885077, max timestamp=1731372885124Num uniques 1027; 2024-11-12T00:54:45,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-12T00:54:45,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-12T00:54:45,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731372885077, max timestamp=1731372885124Num uniques 1027; 2024-11-12T00:54:45,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-12T00:54:45,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731372885197, max timestamp=1731372885248Num uniques 1036; 2024-11-12T00:54:45,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-12T00:54:45,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731372885197, max timestamp=1731372885248Num uniques 1036; 2024-11-12T00:54:45,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-12T00:54:45,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731372885197, max timestamp=1731372885248Num uniques 1036; 2024-11-12T00:54:45,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-12T00:54:45,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731372885336, max timestamp=1731372885371Num uniques 1045; 2024-11-12T00:54:45,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731372885336, max timestamp=1731372885371Num uniques 1045; 2024-11-12T00:54:45,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-12T00:54:45,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-12T00:54:45,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731372885336, max timestamp=1731372885371Num uniques 1045; 2024-11-12T00:54:45,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-12T00:54:45,484 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/534d596c6a0e422fb5868ccbb6069a50 2024-11-12T00:54:45,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/0e03f580a05649b68f06f99492970fb9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0e03f580a05649b68f06f99492970fb9 2024-11-12T00:54:45,491 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0e03f580a05649b68f06f99492970fb9, entries=150, sequenceid=264, filesize=30.5 K 2024-11-12T00:54:45,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/100fe395b70f4051a906f64aa0b9d5af as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/100fe395b70f4051a906f64aa0b9d5af 2024-11-12T00:54:45,495 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/100fe395b70f4051a906f64aa0b9d5af, entries=150, sequenceid=264, filesize=12.0 K 2024-11-12T00:54:45,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/534d596c6a0e422fb5868ccbb6069a50 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/534d596c6a0e422fb5868ccbb6069a50 2024-11-12T00:54:45,499 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/534d596c6a0e422fb5868ccbb6069a50, entries=150, sequenceid=264, filesize=12.0 K 2024-11-12T00:54:45,499 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=127.47 KB/130530 for 58142a01560cfc2d88d5988d1ba12ec1 in 1268ms, sequenceid=264, compaction requested=false 2024-11-12T00:54:45,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:45,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:45,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-12T00:54:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-12T00:54:45,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-12T00:54:45,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6420 sec 2024-11-12T00:54:45,503 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.6460 sec 2024-11-12T00:54:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:45,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=134.18 KB heapSize=381.61 KB 2024-11-12T00:54:45,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:45,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:45,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:45,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:45,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:45,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:54:45,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111206085a9bd03b49529086da05179d8e73_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_0/A:col10/1731372885496/Put/seqid=0 2024-11-12T00:54:45,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742248_1424 (size=14994) 2024-11-12T00:54:45,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372945556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:45,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372945665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:45,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:45,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372945875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:45,914 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:45,917 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111206085a9bd03b49529086da05179d8e73_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111206085a9bd03b49529086da05179d8e73_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:45,917 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/f3b020bd8893475e8cf070b6d744e0b8, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:45,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/f3b020bd8893475e8cf070b6d744e0b8 is 175, key is test_row_0/A:col10/1731372885496/Put/seqid=0 2024-11-12T00:54:45,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742249_1425 (size=39949) 2024-11-12T00:54:46,053 INFO [Thread-1613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1613. 2024-11-12T00:54:46,053 DEBUG [Thread-1613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:46,053 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:46,053 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:46,053 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:46,060 INFO [Thread-1615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1615. 2024-11-12T00:54:46,060 DEBUG [Thread-1615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:46,060 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:46,061 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:46,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:46,062 INFO [Thread-1611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1611. 2024-11-12T00:54:46,062 DEBUG [Thread-1611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:46,062 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:46,062 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:46,062 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:46,064 INFO [Thread-1609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1609. 2024-11-12T00:54:46,064 DEBUG [Thread-1609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:46,064 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:46,064 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:46,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:46,072 INFO [Thread-1607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1607. 2024-11-12T00:54:46,072 DEBUG [Thread-1607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:46,072 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:46,072 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:46,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:46,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372946195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:46,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:46,322 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/f3b020bd8893475e8cf070b6d744e0b8 2024-11-12T00:54:46,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/e7033383d49f4d269050cc0c6a1da79a is 50, key is test_row_0/B:col10/1731372885496/Put/seqid=0 2024-11-12T00:54:46,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742250_1426 (size=12301) 2024-11-12T00:54:46,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:46,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58788 deadline: 1731372946707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:54:46,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/e7033383d49f4d269050cc0c6a1da79a 2024-11-12T00:54:46,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/7267d893fd0d4a5ba1ac1372353dc2ef is 50, key is test_row_0/C:col10/1731372885496/Put/seqid=0 2024-11-12T00:54:46,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742251_1427 (size=12301) 2024-11-12T00:54:47,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T00:54:47,027 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:47,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/7267d893fd0d4a5ba1ac1372353dc2ef 2024-11-12T00:54:47,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/f3b020bd8893475e8cf070b6d744e0b8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/f3b020bd8893475e8cf070b6d744e0b8 2024-11-12T00:54:47,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/f3b020bd8893475e8cf070b6d744e0b8, entries=200, sequenceid=372, filesize=39.0 K 2024-11-12T00:54:47,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/e7033383d49f4d269050cc0c6a1da79a as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/e7033383d49f4d269050cc0c6a1da79a 2024-11-12T00:54:47,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/e7033383d49f4d269050cc0c6a1da79a, entries=150, sequenceid=372, filesize=12.0 K 2024-11-12T00:54:47,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/7267d893fd0d4a5ba1ac1372353dc2ef as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/7267d893fd0d4a5ba1ac1372353dc2ef 2024-11-12T00:54:47,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/7267d893fd0d4a5ba1ac1372353dc2ef, entries=150, sequenceid=372, filesize=12.0 K 2024-11-12T00:54:47,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~381.56 KB/390720, currentSize=46.96 KB/48090 for 58142a01560cfc2d88d5988d1ba12ec1 in 1683ms, sequenceid=372, compaction requested=true 2024-11-12T00:54:47,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:47,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:54:47,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:47,187 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:47,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:54:47,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:47,188 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:47,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58142a01560cfc2d88d5988d1ba12ec1:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:54:47,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:47,188 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:47,188 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:47,188 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/A is initiating minor compaction (all files) 2024-11-12T00:54:47,188 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/B is initiating minor compaction (all files) 2024-11-12T00:54:47,188 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/B in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:47,188 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/A in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:47,188 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/84e37447b17440a1b94c34cf9eece696, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/100fe395b70f4051a906f64aa0b9d5af, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/e7033383d49f4d269050cc0c6a1da79a] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=36.4 K 2024-11-12T00:54:47,188 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0052330284ee447c902dfab6beaf3631, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0e03f580a05649b68f06f99492970fb9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/f3b020bd8893475e8cf070b6d744e0b8] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=100.4 K 2024-11-12T00:54:47,188 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:47,189 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0052330284ee447c902dfab6beaf3631, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0e03f580a05649b68f06f99492970fb9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/f3b020bd8893475e8cf070b6d744e0b8] 2024-11-12T00:54:47,189 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 84e37447b17440a1b94c34cf9eece696, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731372881330 2024-11-12T00:54:47,189 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0052330284ee447c902dfab6beaf3631, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731372881330 2024-11-12T00:54:47,189 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 100fe395b70f4051a906f64aa0b9d5af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731372882556 2024-11-12T00:54:47,189 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e03f580a05649b68f06f99492970fb9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731372882556 2024-11-12T00:54:47,189 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting e7033383d49f4d269050cc0c6a1da79a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731372885469 2024-11-12T00:54:47,189 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3b020bd8893475e8cf070b6d744e0b8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731372885401 2024-11-12T00:54:47,193 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:47,194 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:47,194 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411123050c2d0ad11400db7fa64867cb80b97_58142a01560cfc2d88d5988d1ba12ec1 store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:47,194 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/c8f4d52021e44976b3bef24f2d871ed1 is 50, key is test_row_0/B:col10/1731372885496/Put/seqid=0 2024-11-12T00:54:47,197 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411123050c2d0ad11400db7fa64867cb80b97_58142a01560cfc2d88d5988d1ba12ec1, store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:47,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742252_1428 (size=12949) 2024-11-12T00:54:47,198 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123050c2d0ad11400db7fa64867cb80b97_58142a01560cfc2d88d5988d1ba12ec1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:47,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742253_1429 (size=4469) 2024-11-12T00:54:47,602 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#A#compaction#361 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:47,603 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/6861703315a4492a8d8b9cec7e9225cc is 175, key is test_row_0/A:col10/1731372885496/Put/seqid=0 2024-11-12T00:54:47,604 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/c8f4d52021e44976b3bef24f2d871ed1 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/c8f4d52021e44976b3bef24f2d871ed1 2024-11-12T00:54:47,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742254_1430 (size=31903) 2024-11-12T00:54:47,611 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/B of 58142a01560cfc2d88d5988d1ba12ec1 into c8f4d52021e44976b3bef24f2d871ed1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:47,611 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:47,611 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/B, priority=13, startTime=1731372887187; duration=0sec 2024-11-12T00:54:47,611 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:54:47,611 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:B 2024-11-12T00:54:47,612 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:54:47,613 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:54:47,613 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 58142a01560cfc2d88d5988d1ba12ec1/C is initiating minor compaction (all files) 2024-11-12T00:54:47,613 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58142a01560cfc2d88d5988d1ba12ec1/C in TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:47,613 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/95d819fc0ef04bcdb744dc2e95c5b924, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/534d596c6a0e422fb5868ccbb6069a50, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/7267d893fd0d4a5ba1ac1372353dc2ef] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp, totalSize=36.4 K 2024-11-12T00:54:47,614 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 95d819fc0ef04bcdb744dc2e95c5b924, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731372881330 2024-11-12T00:54:47,615 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 534d596c6a0e422fb5868ccbb6069a50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731372882556 2024-11-12T00:54:47,615 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 7267d893fd0d4a5ba1ac1372353dc2ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731372885469 2024-11-12T00:54:47,623 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58142a01560cfc2d88d5988d1ba12ec1#C#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:54:47,623 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/02334894f38546ffb396d2a1ab18a2c7 is 50, key is test_row_0/C:col10/1731372885496/Put/seqid=0 2024-11-12T00:54:47,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742255_1431 (size=12949) 2024-11-12T00:54:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:47,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T00:54:47,734 INFO [Thread-1598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1598. 2024-11-12T00:54:47,734 DEBUG [Thread-1598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:47,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:47,734 DEBUG [Thread-1598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:47,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:47,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:47,735 DEBUG [Thread-1598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:47,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:47,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:47,735 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:47,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:47,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111233e4f9dec44b4bf1b1e26732be1ff837_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_0/A:col10/1731372885541/Put/seqid=0 2024-11-12T00:54:47,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742256_1432 (size=12454) 2024-11-12T00:54:48,012 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/6861703315a4492a8d8b9cec7e9225cc as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/6861703315a4492a8d8b9cec7e9225cc 2024-11-12T00:54:48,017 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/A of 58142a01560cfc2d88d5988d1ba12ec1 into 6861703315a4492a8d8b9cec7e9225cc(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:48,017 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:48,017 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/A, priority=13, startTime=1731372887187; duration=0sec 2024-11-12T00:54:48,017 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:48,017 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:A 2024-11-12T00:54:48,034 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/02334894f38546ffb396d2a1ab18a2c7 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/02334894f38546ffb396d2a1ab18a2c7 2024-11-12T00:54:48,042 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58142a01560cfc2d88d5988d1ba12ec1/C of 58142a01560cfc2d88d5988d1ba12ec1 into 02334894f38546ffb396d2a1ab18a2c7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:54:48,042 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:48,042 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., storeName=58142a01560cfc2d88d5988d1ba12ec1/C, priority=13, startTime=1731372887188; duration=0sec 2024-11-12T00:54:48,042 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:54:48,042 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58142a01560cfc2d88d5988d1ba12ec1:C 2024-11-12T00:54:48,144 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:48,153 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111233e4f9dec44b4bf1b1e26732be1ff837_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111233e4f9dec44b4bf1b1e26732be1ff837_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:48,154 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/fbc54f5285784e1dabe2211b0065e0f4, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:48,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/fbc54f5285784e1dabe2211b0065e0f4 is 175, key is test_row_0/A:col10/1731372885541/Put/seqid=0 2024-11-12T00:54:48,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742257_1433 (size=31255) 2024-11-12T00:54:48,159 WARN [regionserver/6928b69dca1b:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-12T00:54:48,161 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/3749c4fd64d84b788925b15262425af4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/17fb0abd6ce24188a28caaa9838c7131, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/adcf4b52e80747fea9e613b30e8b3390, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0b6501314ceb4b3a8a836a61fa0067f7, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/e54bdb845a424587b5d58739b84e73d8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/49997e22b91443a691e9a7599ff8ba71, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/7e6ea54c6e114613b96727a0ded93cd8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/78953911d029474d8a0a1922321b1c9b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/90ad4bc90bd9482eab51ebbafb0abc3c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/5d086e15ea9b4218b8bf18276b8b91e5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/62dbf287121943f78af7cbcaf4ddd1e6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/ecd4561d24b9483ca7656f767f7e1bb5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/de96ba41ff784432a418a2c933e38ff2, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/cbb91c2280064e43bf8b5c93b76854c8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/c654aa6012f24b8a8df40ea78a52f47b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/91766d195d32496da8da4fbb8cb089db, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0052330284ee447c902dfab6beaf3631, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0e03f580a05649b68f06f99492970fb9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/f3b020bd8893475e8cf070b6d744e0b8] to archive 2024-11-12T00:54:48,162 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:54:48,164 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/3749c4fd64d84b788925b15262425af4 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/3749c4fd64d84b788925b15262425af4 2024-11-12T00:54:48,165 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/17fb0abd6ce24188a28caaa9838c7131 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/17fb0abd6ce24188a28caaa9838c7131 2024-11-12T00:54:48,167 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/adcf4b52e80747fea9e613b30e8b3390 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/adcf4b52e80747fea9e613b30e8b3390 2024-11-12T00:54:48,168 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0b6501314ceb4b3a8a836a61fa0067f7 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0b6501314ceb4b3a8a836a61fa0067f7 2024-11-12T00:54:48,169 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/e54bdb845a424587b5d58739b84e73d8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/e54bdb845a424587b5d58739b84e73d8 2024-11-12T00:54:48,170 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/49997e22b91443a691e9a7599ff8ba71 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/49997e22b91443a691e9a7599ff8ba71 2024-11-12T00:54:48,172 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/7e6ea54c6e114613b96727a0ded93cd8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/7e6ea54c6e114613b96727a0ded93cd8 2024-11-12T00:54:48,173 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/78953911d029474d8a0a1922321b1c9b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/78953911d029474d8a0a1922321b1c9b 2024-11-12T00:54:48,174 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/90ad4bc90bd9482eab51ebbafb0abc3c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/90ad4bc90bd9482eab51ebbafb0abc3c 2024-11-12T00:54:48,175 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/5d086e15ea9b4218b8bf18276b8b91e5 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/5d086e15ea9b4218b8bf18276b8b91e5 2024-11-12T00:54:48,177 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/62dbf287121943f78af7cbcaf4ddd1e6 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/62dbf287121943f78af7cbcaf4ddd1e6 2024-11-12T00:54:48,178 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/ecd4561d24b9483ca7656f767f7e1bb5 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/ecd4561d24b9483ca7656f767f7e1bb5 2024-11-12T00:54:48,179 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/de96ba41ff784432a418a2c933e38ff2 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/de96ba41ff784432a418a2c933e38ff2 2024-11-12T00:54:48,180 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/cbb91c2280064e43bf8b5c93b76854c8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/cbb91c2280064e43bf8b5c93b76854c8 2024-11-12T00:54:48,181 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/c654aa6012f24b8a8df40ea78a52f47b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/c654aa6012f24b8a8df40ea78a52f47b 2024-11-12T00:54:48,182 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/91766d195d32496da8da4fbb8cb089db to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/91766d195d32496da8da4fbb8cb089db 2024-11-12T00:54:48,182 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0052330284ee447c902dfab6beaf3631 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0052330284ee447c902dfab6beaf3631 2024-11-12T00:54:48,183 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0e03f580a05649b68f06f99492970fb9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/0e03f580a05649b68f06f99492970fb9 2024-11-12T00:54:48,184 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/f3b020bd8893475e8cf070b6d744e0b8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/f3b020bd8893475e8cf070b6d744e0b8 2024-11-12T00:54:48,187 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/2e150201ffef458f9e3f8e7875817fa6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ef5980eeb02645e49c19fa5925601058, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ae4f3f167a134a33b91cce0dd14c2213, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/7e5c2aaed0b346cf9ed5afac583f1e0b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/aca884b3b04c495abd5f5779726cef14, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ad62dd99c5b24c05996d6dfe78ba31dd, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/1ec981a5d8804ed1a931e9e7a4a69b9c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/5ec7db1bc7fc4668b4ea24802e9ade7b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/7a282e6f9eb148528962d2dd1f73225e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/6d4cf723ca254277a6089da03fb9efd0, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/8718ccadf094469c90eee9ccc7f4421c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/960cfa1e199f45cfbc8b5b846611c49b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/10a96256877b414e97e050c3a3e51b7b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/dfbfcbaf8a3246d98a89c5ab0f1cc131, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/192d964b3d094f2783a20342b76cd5d9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/84e37447b17440a1b94c34cf9eece696, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/828bdeb88d454e17b751d18a610ef116, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/100fe395b70f4051a906f64aa0b9d5af, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/e7033383d49f4d269050cc0c6a1da79a] to archive 2024-11-12T00:54:48,188 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:54:48,189 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/2e150201ffef458f9e3f8e7875817fa6 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/2e150201ffef458f9e3f8e7875817fa6 2024-11-12T00:54:48,190 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ef5980eeb02645e49c19fa5925601058 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ef5980eeb02645e49c19fa5925601058 2024-11-12T00:54:48,190 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ae4f3f167a134a33b91cce0dd14c2213 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ae4f3f167a134a33b91cce0dd14c2213 2024-11-12T00:54:48,191 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/7e5c2aaed0b346cf9ed5afac583f1e0b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/7e5c2aaed0b346cf9ed5afac583f1e0b 2024-11-12T00:54:48,192 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/aca884b3b04c495abd5f5779726cef14 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/aca884b3b04c495abd5f5779726cef14 2024-11-12T00:54:48,192 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ad62dd99c5b24c05996d6dfe78ba31dd to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/ad62dd99c5b24c05996d6dfe78ba31dd 2024-11-12T00:54:48,193 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/1ec981a5d8804ed1a931e9e7a4a69b9c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/1ec981a5d8804ed1a931e9e7a4a69b9c 2024-11-12T00:54:48,194 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/5ec7db1bc7fc4668b4ea24802e9ade7b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/5ec7db1bc7fc4668b4ea24802e9ade7b 2024-11-12T00:54:48,194 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/7a282e6f9eb148528962d2dd1f73225e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/7a282e6f9eb148528962d2dd1f73225e 2024-11-12T00:54:48,195 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/6d4cf723ca254277a6089da03fb9efd0 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/6d4cf723ca254277a6089da03fb9efd0 2024-11-12T00:54:48,196 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/8718ccadf094469c90eee9ccc7f4421c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/8718ccadf094469c90eee9ccc7f4421c 2024-11-12T00:54:48,196 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/960cfa1e199f45cfbc8b5b846611c49b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/960cfa1e199f45cfbc8b5b846611c49b 2024-11-12T00:54:48,197 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/10a96256877b414e97e050c3a3e51b7b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/10a96256877b414e97e050c3a3e51b7b 2024-11-12T00:54:48,198 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/dfbfcbaf8a3246d98a89c5ab0f1cc131 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/dfbfcbaf8a3246d98a89c5ab0f1cc131 2024-11-12T00:54:48,198 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/192d964b3d094f2783a20342b76cd5d9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/192d964b3d094f2783a20342b76cd5d9 2024-11-12T00:54:48,199 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/84e37447b17440a1b94c34cf9eece696 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/84e37447b17440a1b94c34cf9eece696 2024-11-12T00:54:48,200 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/828bdeb88d454e17b751d18a610ef116 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/828bdeb88d454e17b751d18a610ef116 2024-11-12T00:54:48,200 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/100fe395b70f4051a906f64aa0b9d5af to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/100fe395b70f4051a906f64aa0b9d5af 2024-11-12T00:54:48,201 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/e7033383d49f4d269050cc0c6a1da79a to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/e7033383d49f4d269050cc0c6a1da79a 2024-11-12T00:54:48,204 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/96da969663aa461c936e7c8cd2ba371c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/dd35c26a3c024071b739244ec434e66b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/eb6e2b318715456887e80c5eb9750321, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/948514d1e56d40dcbaebaf1762addfa2, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/1d96ff2b4f854eafba996a0c8d6754c8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/73a4647d282a4ec0a628a3695e629e9f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/acb01ecad3974545bb7d470686d0be5a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/82860a4baea24f56976702449a6c23d9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/391d5e9adf5e48b08d85a356d0947625, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/2bef979dedc848dd97730e84f42f1120, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/f3550b08dab44e23a0b4ae6f17ba9633, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/ee2705072f134dd1b3baa3d58dc97638, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/379ae6d0e5174a4890cd6bd22dbb8982, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/4ea6633b642748dba6ab235ce94d8366, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/02013c9ea6b04c218f20b4a77ec57692, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/95d819fc0ef04bcdb744dc2e95c5b924, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/2f3e862f33ff4901bdea78f47a0078c8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/534d596c6a0e422fb5868ccbb6069a50, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/7267d893fd0d4a5ba1ac1372353dc2ef] to archive 2024-11-12T00:54:48,204 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:54:48,205 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/96da969663aa461c936e7c8cd2ba371c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/96da969663aa461c936e7c8cd2ba371c 2024-11-12T00:54:48,207 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/dd35c26a3c024071b739244ec434e66b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/dd35c26a3c024071b739244ec434e66b 2024-11-12T00:54:48,208 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/eb6e2b318715456887e80c5eb9750321 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/eb6e2b318715456887e80c5eb9750321 2024-11-12T00:54:48,209 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/948514d1e56d40dcbaebaf1762addfa2 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/948514d1e56d40dcbaebaf1762addfa2 2024-11-12T00:54:48,209 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/1d96ff2b4f854eafba996a0c8d6754c8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/1d96ff2b4f854eafba996a0c8d6754c8 2024-11-12T00:54:48,210 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/73a4647d282a4ec0a628a3695e629e9f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/73a4647d282a4ec0a628a3695e629e9f 2024-11-12T00:54:48,211 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/acb01ecad3974545bb7d470686d0be5a to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/acb01ecad3974545bb7d470686d0be5a 2024-11-12T00:54:48,212 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/82860a4baea24f56976702449a6c23d9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/82860a4baea24f56976702449a6c23d9 2024-11-12T00:54:48,213 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/391d5e9adf5e48b08d85a356d0947625 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/391d5e9adf5e48b08d85a356d0947625 2024-11-12T00:54:48,214 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/2bef979dedc848dd97730e84f42f1120 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/2bef979dedc848dd97730e84f42f1120 2024-11-12T00:54:48,215 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/f3550b08dab44e23a0b4ae6f17ba9633 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/f3550b08dab44e23a0b4ae6f17ba9633 2024-11-12T00:54:48,216 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/ee2705072f134dd1b3baa3d58dc97638 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/ee2705072f134dd1b3baa3d58dc97638 2024-11-12T00:54:48,216 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/379ae6d0e5174a4890cd6bd22dbb8982 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/379ae6d0e5174a4890cd6bd22dbb8982 2024-11-12T00:54:48,217 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/4ea6633b642748dba6ab235ce94d8366 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/4ea6633b642748dba6ab235ce94d8366 2024-11-12T00:54:48,218 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/02013c9ea6b04c218f20b4a77ec57692 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/02013c9ea6b04c218f20b4a77ec57692 2024-11-12T00:54:48,219 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/95d819fc0ef04bcdb744dc2e95c5b924 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/95d819fc0ef04bcdb744dc2e95c5b924 2024-11-12T00:54:48,220 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/2f3e862f33ff4901bdea78f47a0078c8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/2f3e862f33ff4901bdea78f47a0078c8 2024-11-12T00:54:48,221 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/534d596c6a0e422fb5868ccbb6069a50 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/534d596c6a0e422fb5868ccbb6069a50 2024-11-12T00:54:48,222 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/6928b69dca1b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/7267d893fd0d4a5ba1ac1372353dc2ef to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/7267d893fd0d4a5ba1ac1372353dc2ef 2024-11-12T00:54:48,560 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/fbc54f5285784e1dabe2211b0065e0f4 2024-11-12T00:54:48,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/a159bae546e64d9fa4b57a35aa725d19 is 50, key is test_row_0/B:col10/1731372885541/Put/seqid=0 2024-11-12T00:54:48,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742258_1434 (size=12301) 2024-11-12T00:54:48,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/a159bae546e64d9fa4b57a35aa725d19 2024-11-12T00:54:48,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/60236f1f27bd4b79afdf396b8b778cfb is 50, key is test_row_0/C:col10/1731372885541/Put/seqid=0 2024-11-12T00:54:48,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742259_1435 (size=12301) 2024-11-12T00:54:49,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/60236f1f27bd4b79afdf396b8b778cfb 2024-11-12T00:54:49,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/fbc54f5285784e1dabe2211b0065e0f4 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/fbc54f5285784e1dabe2211b0065e0f4 2024-11-12T00:54:49,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/fbc54f5285784e1dabe2211b0065e0f4, entries=150, sequenceid=384, filesize=30.5 K 2024-11-12T00:54:49,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/a159bae546e64d9fa4b57a35aa725d19 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/a159bae546e64d9fa4b57a35aa725d19 2024-11-12T00:54:49,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/a159bae546e64d9fa4b57a35aa725d19, entries=150, sequenceid=384, filesize=12.0 K 2024-11-12T00:54:49,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/60236f1f27bd4b79afdf396b8b778cfb as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/60236f1f27bd4b79afdf396b8b778cfb 2024-11-12T00:54:49,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/60236f1f27bd4b79afdf396b8b778cfb, entries=150, sequenceid=384, filesize=12.0 K 2024-11-12T00:54:49,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 58142a01560cfc2d88d5988d1ba12ec1 in 1683ms, sequenceid=384, compaction requested=false 2024-11-12T00:54:49,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58142a01560cfc2d88d5988d1ba12ec1: 2024-11-12T00:54:54,091 INFO [Thread-1602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1602. 2024-11-12T00:54:54,091 DEBUG [Thread-1602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:54,091 DEBUG [Thread-1602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:54,091 DEBUG [Thread-1602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:54,091 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:54,131 INFO [Thread-1596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1596. 2024-11-12T00:54:54,131 INFO [Thread-1604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1604. 2024-11-12T00:54:54,131 DEBUG [Thread-1604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:54,131 DEBUG [Thread-1596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:54,131 DEBUG [Thread-1604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:54,131 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:54,131 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:54,131 DEBUG [Thread-1604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:54,131 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:54,131 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:54,137 INFO [Thread-1600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1600. 2024-11-12T00:54:54,137 DEBUG [Thread-1600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:54:54,137 DEBUG [Thread-1600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:54,137 DEBUG [Thread-1600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:54,137 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:54,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T00:54:54,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-12T00:54:54,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 220 2024-11-12T00:54:54,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-12T00:54:54,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-12T00:54:54,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-12T00:54:54,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T00:54:54,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T00:54:54,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1191 2024-11-12T00:54:54,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3573 rows 2024-11-12T00:54:54,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1195 2024-11-12T00:54:54,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3582 rows 2024-11-12T00:54:54,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1198 2024-11-12T00:54:54,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3593 rows 2024-11-12T00:54:54,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1191 2024-11-12T00:54:54,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3565 rows 2024-11-12T00:54:54,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1196 2024-11-12T00:54:54,138 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3587 rows 2024-11-12T00:54:54,138 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T00:54:54,138 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T00:54:54,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:54,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:54,138 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:54:54,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T00:54:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T00:54:54,141 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372894141"}]},"ts":"1731372894141"} 2024-11-12T00:54:54,142 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T00:54:54,142 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T00:54:54,143 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T00:54:54,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58142a01560cfc2d88d5988d1ba12ec1, UNASSIGN}] 2024-11-12T00:54:54,145 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58142a01560cfc2d88d5988d1ba12ec1, UNASSIGN 2024-11-12T00:54:54,145 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=58142a01560cfc2d88d5988d1ba12ec1, regionState=CLOSING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:54,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58142a01560cfc2d88d5988d1ba12ec1, UNASSIGN because future has completed 2024-11-12T00:54:54,146 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T00:54:54,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:54:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T00:54:54,300 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(122): Close 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:54,300 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T00:54:54,300 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1722): Closing 58142a01560cfc2d88d5988d1ba12ec1, disabling compactions & flushes 2024-11-12T00:54:54,300 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:54,300 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:54,300 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. after waiting 0 ms 2024-11-12T00:54:54,300 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:54,301 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(2902): Flushing 58142a01560cfc2d88d5988d1ba12ec1 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T00:54:54,301 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=A 2024-11-12T00:54:54,301 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:54,301 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=B 2024-11-12T00:54:54,302 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:54,302 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58142a01560cfc2d88d5988d1ba12ec1, store=C 2024-11-12T00:54:54,302 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:54,311 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112cb21c0eb769643278bf37571c6cecd80_58142a01560cfc2d88d5988d1ba12ec1 is 50, key is test_row_1/A:col10/1731372894136/Put/seqid=0 2024-11-12T00:54:54,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742260_1436 (size=9914) 2024-11-12T00:54:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T00:54:54,717 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:54,729 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112cb21c0eb769643278bf37571c6cecd80_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112cb21c0eb769643278bf37571c6cecd80_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:54,730 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/65ba39de44684422873455219ba8cdea, store: [table=TestAcidGuarantees family=A region=58142a01560cfc2d88d5988d1ba12ec1] 2024-11-12T00:54:54,730 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/65ba39de44684422873455219ba8cdea is 175, key is test_row_1/A:col10/1731372894136/Put/seqid=0 2024-11-12T00:54:54,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742261_1437 (size=22561) 2024-11-12T00:54:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T00:54:55,135 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/65ba39de44684422873455219ba8cdea 2024-11-12T00:54:55,147 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/51493ff7a2ba49fb8b62249645762ade is 50, key is test_row_1/B:col10/1731372894136/Put/seqid=0 2024-11-12T00:54:55,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742262_1438 (size=9857) 2024-11-12T00:54:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T00:54:55,551 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/51493ff7a2ba49fb8b62249645762ade 2024-11-12T00:54:55,562 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/b258e5cd1c2e424ebb734f7bc5bf3423 is 50, key is test_row_1/C:col10/1731372894136/Put/seqid=0 2024-11-12T00:54:55,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742263_1439 (size=9857) 2024-11-12T00:54:55,968 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/b258e5cd1c2e424ebb734f7bc5bf3423 2024-11-12T00:54:55,976 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/A/65ba39de44684422873455219ba8cdea as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/65ba39de44684422873455219ba8cdea 2024-11-12T00:54:55,982 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/65ba39de44684422873455219ba8cdea, entries=100, sequenceid=393, filesize=22.0 K 2024-11-12T00:54:55,983 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/B/51493ff7a2ba49fb8b62249645762ade as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/51493ff7a2ba49fb8b62249645762ade 2024-11-12T00:54:55,988 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/51493ff7a2ba49fb8b62249645762ade, entries=100, sequenceid=393, filesize=9.6 K 2024-11-12T00:54:55,988 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/.tmp/C/b258e5cd1c2e424ebb734f7bc5bf3423 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/b258e5cd1c2e424ebb734f7bc5bf3423 2024-11-12T00:54:55,993 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/b258e5cd1c2e424ebb734f7bc5bf3423, entries=100, sequenceid=393, filesize=9.6 K 2024-11-12T00:54:55,994 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 58142a01560cfc2d88d5988d1ba12ec1 in 1694ms, sequenceid=393, compaction requested=true 2024-11-12T00:54:56,000 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/recovered.edits/396.seqid, newMaxSeqId=396, maxSeqId=4 2024-11-12T00:54:56,001 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. 2024-11-12T00:54:56,001 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1676): Region close journal for 58142a01560cfc2d88d5988d1ba12ec1: Waiting for close lock at 1731372894300Running coprocessor pre-close hooks at 1731372894300Disabling compacts and flushes for region at 1731372894300Disabling writes for close at 1731372894300Obtaining lock to block concurrent updates at 1731372894301 (+1 ms)Preparing flush snapshotting stores in 58142a01560cfc2d88d5988d1ba12ec1 at 1731372894301Finished memstore snapshotting TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731372894302 (+1 ms)Flushing stores of TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1. at 1731372894303 (+1 ms)Flushing 58142a01560cfc2d88d5988d1ba12ec1/A: creating writer at 1731372894303Flushing mob file 58142a01560cfc2d88d5988d1ba12ec1/A: appending metadata at 1731372894311 (+8 ms)Flushing mob file 58142a01560cfc2d88d5988d1ba12ec1/A: closing flushed file at 1731372894311Flushing 58142a01560cfc2d88d5988d1ba12ec1/A: appending metadata at 1731372894730 (+419 ms)Flushing 58142a01560cfc2d88d5988d1ba12ec1/A: closing flushed file at 1731372894730Flushing 58142a01560cfc2d88d5988d1ba12ec1/B: creating writer at 1731372895144 (+414 ms)Flushing 58142a01560cfc2d88d5988d1ba12ec1/B: appending metadata at 1731372895146 (+2 ms)Flushing 58142a01560cfc2d88d5988d1ba12ec1/B: closing flushed file at 1731372895146Flushing 58142a01560cfc2d88d5988d1ba12ec1/C: creating writer at 1731372895557 (+411 ms)Flushing 58142a01560cfc2d88d5988d1ba12ec1/C: appending metadata at 1731372895561 (+4 ms)Flushing 58142a01560cfc2d88d5988d1ba12ec1/C: closing flushed file at 1731372895562 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ccef774: reopening flushed file at 1731372895975 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b281ecc: reopening flushed file at 1731372895982 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14214fbd: reopening flushed file at 1731372895988 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 58142a01560cfc2d88d5988d1ba12ec1 in 1694ms, sequenceid=393, compaction requested=true at 1731372895994 (+6 ms)Writing region close event to WAL at 1731372895997 (+3 ms)Running coprocessor post-close hooks at 1731372896001 (+4 ms)Closed at 1731372896001 2024-11-12T00:54:56,004 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(157): Closed 58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,004 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=58142a01560cfc2d88d5988d1ba12ec1, regionState=CLOSED 2024-11-12T00:54:56,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:54:56,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-12T00:54:56,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseRegionProcedure 58142a01560cfc2d88d5988d1ba12ec1, server=6928b69dca1b,33953,1731372765559 in 1.8610 sec 2024-11-12T00:54:56,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-12T00:54:56,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58142a01560cfc2d88d5988d1ba12ec1, UNASSIGN in 1.8660 sec 2024-11-12T00:54:56,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-12T00:54:56,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8700 sec 2024-11-12T00:54:56,017 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372896017"}]},"ts":"1731372896017"} 2024-11-12T00:54:56,019 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T00:54:56,019 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T00:54:56,021 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8810 sec 2024-11-12T00:54:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T00:54:56,287 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:56,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T00:54:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:56,293 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:56,295 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=105, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T00:54:56,299 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,302 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/recovered.edits] 2024-11-12T00:54:56,305 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/65ba39de44684422873455219ba8cdea to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/65ba39de44684422873455219ba8cdea 2024-11-12T00:54:56,307 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/6861703315a4492a8d8b9cec7e9225cc to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/6861703315a4492a8d8b9cec7e9225cc 2024-11-12T00:54:56,309 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/fbc54f5285784e1dabe2211b0065e0f4 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/A/fbc54f5285784e1dabe2211b0065e0f4 2024-11-12T00:54:56,310 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/51493ff7a2ba49fb8b62249645762ade to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/51493ff7a2ba49fb8b62249645762ade 2024-11-12T00:54:56,311 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/a159bae546e64d9fa4b57a35aa725d19 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/a159bae546e64d9fa4b57a35aa725d19 2024-11-12T00:54:56,312 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/c8f4d52021e44976b3bef24f2d871ed1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/B/c8f4d52021e44976b3bef24f2d871ed1 2024-11-12T00:54:56,313 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/02334894f38546ffb396d2a1ab18a2c7 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/02334894f38546ffb396d2a1ab18a2c7 2024-11-12T00:54:56,315 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/60236f1f27bd4b79afdf396b8b778cfb to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/60236f1f27bd4b79afdf396b8b778cfb 2024-11-12T00:54:56,316 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/b258e5cd1c2e424ebb734f7bc5bf3423 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/C/b258e5cd1c2e424ebb734f7bc5bf3423 2024-11-12T00:54:56,318 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/recovered.edits/396.seqid to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1/recovered.edits/396.seqid 2024-11-12T00:54:56,319 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,319 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T00:54:56,319 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T00:54:56,320 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T00:54:56,323 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111206085a9bd03b49529086da05179d8e73_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111206085a9bd03b49529086da05179d8e73_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,324 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111233e4f9dec44b4bf1b1e26732be1ff837_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111233e4f9dec44b4bf1b1e26732be1ff837_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,325 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111248b721fc41e746fda9676738da274d8f_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111248b721fc41e746fda9676738da274d8f_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,326 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125a02da066bea4841a234176769ce31ed_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125a02da066bea4841a234176769ce31ed_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,327 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112615a15e8269f4a9db6d06ad8d7668e60_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112615a15e8269f4a9db6d06ad8d7668e60_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,328 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112777fc906ee314b989cab93b3b9e96df4_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112777fc906ee314b989cab93b3b9e96df4_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,329 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127ef197c0904741a4b8e2bc87aed2c4f8_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127ef197c0904741a4b8e2bc87aed2c4f8_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,330 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b2f6240023424499a3f5c2863e4752b9_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b2f6240023424499a3f5c2863e4752b9_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,332 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b3d023b7c8634050ba385464054d61b0_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b3d023b7c8634050ba385464054d61b0_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,333 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b4f0f5f252e342d090cbe4dc6f3790de_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b4f0f5f252e342d090cbe4dc6f3790de_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,335 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c2bca635964e46b9b2ab773ffd81a8e6_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c2bca635964e46b9b2ab773ffd81a8e6_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,336 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c71eb4c8cf024db4b667ffbab728585b_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c71eb4c8cf024db4b667ffbab728585b_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,336 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112caf28e64b05a4959b4ce8f96abd484c0_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112caf28e64b05a4959b4ce8f96abd484c0_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,337 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112cb21c0eb769643278bf37571c6cecd80_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112cb21c0eb769643278bf37571c6cecd80_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,338 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ecd61f8d743942f3a6f52fb83103517f_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ecd61f8d743942f3a6f52fb83103517f_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,339 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f95e286395784a50aefbdfeab7f665ee_58142a01560cfc2d88d5988d1ba12ec1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f95e286395784a50aefbdfeab7f665ee_58142a01560cfc2d88d5988d1ba12ec1 2024-11-12T00:54:56,339 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T00:54:56,341 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=105, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:56,343 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T00:54:56,345 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T00:54:56,346 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=105, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:56,346 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T00:54:56,346 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731372896346"}]},"ts":"9223372036854775807"} 2024-11-12T00:54:56,348 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T00:54:56,348 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 58142a01560cfc2d88d5988d1ba12ec1, NAME => 'TestAcidGuarantees,,1731372862903.58142a01560cfc2d88d5988d1ba12ec1.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T00:54:56,348 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T00:54:56,348 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731372896348"}]},"ts":"9223372036854775807"} 2024-11-12T00:54:56,350 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T00:54:56,351 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=105, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:56,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-11-12T00:54:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T00:54:56,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T00:54:56,406 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:56,414 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 247) - Thread LEAK? -, OpenFileDescriptor=445 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=385 (was 450), ProcessCount=11 (was 11), AvailableMemoryMB=5102 (was 5229) 2024-11-12T00:54:56,422 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=385, ProcessCount=11, AvailableMemoryMB=5101 2024-11-12T00:54:56,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T00:54:56,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T00:54:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T00:54:56,425 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T00:54:56,425 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:56,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 106 2024-11-12T00:54:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-12T00:54:56,426 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T00:54:56,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742264_1440 (size=1061) 2024-11-12T00:54:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-12T00:54:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-12T00:54:56,834 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6 2024-11-12T00:54:56,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742265_1441 (size=53) 2024-11-12T00:54:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-12T00:54:57,245 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:54:57,245 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2400f2d4915c3c25d637e8ed29b4fadd, disabling compactions & flushes 2024-11-12T00:54:57,245 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:57,245 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:57,245 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. after waiting 0 ms 2024-11-12T00:54:57,245 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:57,245 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:57,245 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2400f2d4915c3c25d637e8ed29b4fadd: Waiting for close lock at 1731372897245Disabling compacts and flushes for region at 1731372897245Disabling writes for close at 1731372897245Writing region close event to WAL at 1731372897245Closed at 1731372897245 2024-11-12T00:54:57,248 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T00:54:57,249 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731372897248"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731372897248"}]},"ts":"1731372897248"} 2024-11-12T00:54:57,253 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T00:54:57,253 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T00:54:57,254 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372897253"}]},"ts":"1731372897253"} 2024-11-12T00:54:57,255 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T00:54:57,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2400f2d4915c3c25d637e8ed29b4fadd, ASSIGN}] 2024-11-12T00:54:57,256 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2400f2d4915c3c25d637e8ed29b4fadd, ASSIGN 2024-11-12T00:54:57,256 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2400f2d4915c3c25d637e8ed29b4fadd, ASSIGN; state=OFFLINE, location=6928b69dca1b,33953,1731372765559; forceNewPlan=false, retain=false 2024-11-12T00:54:57,407 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=2400f2d4915c3c25d637e8ed29b4fadd, regionState=OPENING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:57,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2400f2d4915c3c25d637e8ed29b4fadd, ASSIGN because future has completed 2024-11-12T00:54:57,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:54:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-12T00:54:57,566 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:57,566 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} 2024-11-12T00:54:57,567 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:54:57,567 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:54:57,567 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:54:57,567 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:54:57,569 INFO [StoreOpener-2400f2d4915c3c25d637e8ed29b4fadd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:54:57,571 INFO [StoreOpener-2400f2d4915c3c25d637e8ed29b4fadd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:54:57,571 INFO [StoreOpener-2400f2d4915c3c25d637e8ed29b4fadd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2400f2d4915c3c25d637e8ed29b4fadd columnFamilyName A 2024-11-12T00:54:57,571 DEBUG [StoreOpener-2400f2d4915c3c25d637e8ed29b4fadd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:57,572 INFO [StoreOpener-2400f2d4915c3c25d637e8ed29b4fadd-1 {}] regionserver.HStore(327): Store=2400f2d4915c3c25d637e8ed29b4fadd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:54:57,572 INFO [StoreOpener-2400f2d4915c3c25d637e8ed29b4fadd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:54:57,573 INFO [StoreOpener-2400f2d4915c3c25d637e8ed29b4fadd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:54:57,574 INFO [StoreOpener-2400f2d4915c3c25d637e8ed29b4fadd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2400f2d4915c3c25d637e8ed29b4fadd columnFamilyName B 2024-11-12T00:54:57,574 DEBUG [StoreOpener-2400f2d4915c3c25d637e8ed29b4fadd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:57,574 INFO [StoreOpener-2400f2d4915c3c25d637e8ed29b4fadd-1 {}] regionserver.HStore(327): Store=2400f2d4915c3c25d637e8ed29b4fadd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:54:57,574 INFO [StoreOpener-2400f2d4915c3c25d637e8ed29b4fadd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:54:57,576 INFO [StoreOpener-2400f2d4915c3c25d637e8ed29b4fadd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:54:57,576 INFO [StoreOpener-2400f2d4915c3c25d637e8ed29b4fadd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2400f2d4915c3c25d637e8ed29b4fadd columnFamilyName C 2024-11-12T00:54:57,576 DEBUG [StoreOpener-2400f2d4915c3c25d637e8ed29b4fadd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:54:57,576 INFO [StoreOpener-2400f2d4915c3c25d637e8ed29b4fadd-1 {}] regionserver.HStore(327): Store=2400f2d4915c3c25d637e8ed29b4fadd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:54:57,577 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:57,577 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:54:57,577 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:54:57,578 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:54:57,579 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:54:57,579 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:54:57,580 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T00:54:57,581 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:54:57,583 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T00:54:57,584 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened 2400f2d4915c3c25d637e8ed29b4fadd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73341643, jitterRate=0.0928756445646286}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T00:54:57,584 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:54:57,584 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for 2400f2d4915c3c25d637e8ed29b4fadd: Running coprocessor pre-open hook at 1731372897567Writing region info on filesystem at 1731372897568 (+1 ms)Initializing all the Stores at 1731372897568Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372897569 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372897569Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372897569Cleaning up temporary data from old regions at 1731372897579 (+10 ms)Running coprocessor post-open hooks at 1731372897584 (+5 ms)Region opened successfully at 1731372897584 2024-11-12T00:54:57,585 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., pid=108, masterSystemTime=1731372897562 2024-11-12T00:54:57,587 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:57,587 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:57,587 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=2400f2d4915c3c25d637e8ed29b4fadd, regionState=OPEN, openSeqNum=2, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:57,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:54:57,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-12T00:54:57,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; OpenRegionProcedure 2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 in 178 msec 2024-11-12T00:54:57,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-12T00:54:57,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2400f2d4915c3c25d637e8ed29b4fadd, ASSIGN in 335 msec 2024-11-12T00:54:57,591 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T00:54:57,591 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372897591"}]},"ts":"1731372897591"} 2024-11-12T00:54:57,592 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T00:54:57,593 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T00:54:57,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-11-12T00:54:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-12T00:54:58,576 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:54:58,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3005a7e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:58,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:58,578 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a161d8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:58,580 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:58,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4429de15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,582 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:58,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:58,584 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:58,584 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T00:54:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 6928b69dca1b,45925,1731372764943 2024-11-12T00:54:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5569f552 2024-11-12T00:54:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T00:54:58,586 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T00:54:58,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72417d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:58,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:58,589 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c744f3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:58,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:58,590 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f418403, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:58,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:58,593 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af5cf1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:58,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:58,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:58,594 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79708568, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:58,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:58,597 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@639ba136, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:58,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:58,598 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:58,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37adf233, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,600 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:58,600 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:58,601 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:58,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:58,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:58,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec3f93c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:58,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:58,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:58,603 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:58,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d52ed6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,605 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:58,605 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:58,606 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69caa0de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:58,607 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8b6806, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:58,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:58,609 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2539be30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:58,610 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:58,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cb1df3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,611 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:58,611 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:58,612 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eb09731, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:58,613 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:58,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3766943d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:58,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:58,616 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ce7f016, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:58,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:58,617 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3846925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,618 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:58,618 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:58,619 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16395946, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:58,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:58,620 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:58,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40cc350b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:54:58,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:54:58,622 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:54:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:54:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:54:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c97db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:54:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:54:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:54:58,623 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:54:58,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35411270, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:54:58,624 DEBUG [Thread-1998 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:58,624 DEBUG [Thread-1985 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:58,624 DEBUG [Thread-1989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:58,625 DEBUG [Thread-2002 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:58,625 DEBUG [Thread-1987 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:58,625 DEBUG [Thread-1996 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:58,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:58,626 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:54:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:58,626 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:58,626 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:58,626 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:58,626 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:58,626 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:58,627 DEBUG [Thread-1993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:58,627 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:58,627 DEBUG [Thread-1991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:58,627 DEBUG [Thread-2004 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:58,627 DEBUG [Thread-2000 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:54:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-12T00:54:58,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:58,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:54:58,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:58,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:54:58,629 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:54:58,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:54:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:54:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:54:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:54:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T00:54:58,631 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:54:58,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:58,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:54:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:54:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:54:58,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:54:58,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:54:58,634 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:54:58,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:54:58,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2] 2024-11-12T00:54:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:54:58,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T00:54:58,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:54:58,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:58,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:54:58,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:58,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:54:58,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:58,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/18bbb76ed13a43fbb98309ef36ad9422 is 50, key is test_row_0/A:col10/1731372898629/Put/seqid=0 2024-11-12T00:54:58,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742266_1442 (size=14341) 2024-11-12T00:54:58,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/18bbb76ed13a43fbb98309ef36ad9422 2024-11-12T00:54:58,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372958663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:58,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39678 deadline: 1731372958664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:58,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372958669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:58,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372958671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:58,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372958673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:58,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/a09c4b85321246bf85ec9c416cf61474 is 50, key is test_row_0/B:col10/1731372898629/Put/seqid=0 2024-11-12T00:54:58,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742267_1443 (size=12001) 2024-11-12T00:54:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T00:54:58,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372958775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:58,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T00:54:58,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:58,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:54:58,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:58,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:58,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:58,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372958786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372958786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:58,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39678 deadline: 1731372958786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372958786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:58,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T00:54:58,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:58,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:54:58,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:58,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:58,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T00:54:58,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372958986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372958995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372958995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39678 deadline: 1731372958995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372958995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T00:54:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:54:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:59,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:59,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:59,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/a09c4b85321246bf85ec9c416cf61474 2024-11-12T00:54:59,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/1ce125510b234f38afd958b71bb2723e is 50, key is test_row_0/C:col10/1731372898629/Put/seqid=0 2024-11-12T00:54:59,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742268_1444 (size=12001) 2024-11-12T00:54:59,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T00:54:59,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:59,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:54:59,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:59,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:59,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T00:54:59,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372959296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372959306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372959306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39678 deadline: 1731372959306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372959306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T00:54:59,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:59,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:54:59,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:59,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:59,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:54:59,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/1ce125510b234f38afd958b71bb2723e 2024-11-12T00:54:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/18bbb76ed13a43fbb98309ef36ad9422 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/18bbb76ed13a43fbb98309ef36ad9422 2024-11-12T00:54:59,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/18bbb76ed13a43fbb98309ef36ad9422, entries=200, sequenceid=14, filesize=14.0 K 2024-11-12T00:54:59,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/a09c4b85321246bf85ec9c416cf61474 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a09c4b85321246bf85ec9c416cf61474 2024-11-12T00:54:59,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a09c4b85321246bf85ec9c416cf61474, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T00:54:59,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/1ce125510b234f38afd958b71bb2723e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/1ce125510b234f38afd958b71bb2723e 2024-11-12T00:54:59,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/1ce125510b234f38afd958b71bb2723e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T00:54:59,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2400f2d4915c3c25d637e8ed29b4fadd in 895ms, sequenceid=14, compaction requested=false 2024-11-12T00:54:59,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:54:59,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T00:54:59,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:54:59,548 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T00:54:59,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:54:59,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:59,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:54:59,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:59,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:54:59,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:54:59,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/a42839bd7b724c82ba1fc9eb19487bd1 is 50, key is test_row_0/A:col10/1731372898671/Put/seqid=0 2024-11-12T00:54:59,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742269_1445 (size=12001) 2024-11-12T00:54:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T00:54:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:54:59,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:54:59,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39678 deadline: 1731372959825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372959826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372959831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372959832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372959832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372959935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372959946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372959946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:54:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372959946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:54:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:54:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:54:59,955 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/a42839bd7b724c82ba1fc9eb19487bd1 2024-11-12T00:54:59,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/85000c30178e49b3895e7af38e559f82 is 50, key is test_row_0/B:col10/1731372898671/Put/seqid=0 2024-11-12T00:54:59,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742270_1446 (size=12001) 2024-11-12T00:54:59,978 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/85000c30178e49b3895e7af38e559f82 2024-11-12T00:54:59,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/6a8d31e140554b39a6b1556fb682ee87 is 50, key is test_row_0/C:col10/1731372898671/Put/seqid=0 2024-11-12T00:55:00,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742271_1447 (size=12001) 2024-11-12T00:55:00,001 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/6a8d31e140554b39a6b1556fb682ee87 2024-11-12T00:55:00,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/a42839bd7b724c82ba1fc9eb19487bd1 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/a42839bd7b724c82ba1fc9eb19487bd1 2024-11-12T00:55:00,008 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/a42839bd7b724c82ba1fc9eb19487bd1, entries=150, sequenceid=33, filesize=11.7 K 2024-11-12T00:55:00,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/85000c30178e49b3895e7af38e559f82 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/85000c30178e49b3895e7af38e559f82 2024-11-12T00:55:00,012 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/85000c30178e49b3895e7af38e559f82, entries=150, sequenceid=33, filesize=11.7 K 2024-11-12T00:55:00,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/6a8d31e140554b39a6b1556fb682ee87 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/6a8d31e140554b39a6b1556fb682ee87 2024-11-12T00:55:00,015 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/6a8d31e140554b39a6b1556fb682ee87, entries=150, sequenceid=33, filesize=11.7 K 2024-11-12T00:55:00,016 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 2400f2d4915c3c25d637e8ed29b4fadd in 468ms, sequenceid=33, compaction requested=false 2024-11-12T00:55:00,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:00,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:00,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-12T00:55:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-12T00:55:00,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-12T00:55:00,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3830 sec 2024-11-12T00:55:00,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.3930 sec 2024-11-12T00:55:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:00,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T00:55:00,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:00,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:00,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:00,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:00,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:00,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:00,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/0630ff65950e4c4f9bc151825af1bd32 is 50, key is test_row_0/A:col10/1731372899830/Put/seqid=0 2024-11-12T00:55:00,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742272_1448 (size=14341) 2024-11-12T00:55:00,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372960196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372960196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372960197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372960198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372960315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372960315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372960315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372960316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372960525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372960535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372960535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372960536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/0630ff65950e4c4f9bc151825af1bd32 2024-11-12T00:55:00,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/7766fbf116c94964b023c3a83c5ab848 is 50, key is test_row_0/B:col10/1731372899830/Put/seqid=0 2024-11-12T00:55:00,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742273_1449 (size=12001) 2024-11-12T00:55:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T00:55:00,776 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:00,776 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:55:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-12T00:55:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T00:55:00,777 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:55:00,778 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:55:00,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:55:00,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372960836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39678 deadline: 1731372960845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372960845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372960845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372960845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,853 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 2400f2d4915c3c25d637e8ed29b4fadd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T00:55:00,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-12T00:55:00,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:00,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:00,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:00,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:00,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:00,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/7766fbf116c94964b023c3a83c5ab848 2024-11-12T00:55:00,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/519615f5e9a443abaded6a44fbb8ac1e is 50, key is test_row_0/C:col10/1731372899830/Put/seqid=0 2024-11-12T00:55:00,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742274_1450 (size=12001) 2024-11-12T00:55:01,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-12T00:55:01,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:01,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:01,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:01,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:01,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T00:55:01,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-12T00:55:01,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:01,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:01,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:01,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:01,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:01,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372961345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:01,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:01,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:01,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:01,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372961366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:01,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:01,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372961366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372961366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:01,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/519615f5e9a443abaded6a44fbb8ac1e 2024-11-12T00:55:01,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/0630ff65950e4c4f9bc151825af1bd32 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/0630ff65950e4c4f9bc151825af1bd32 2024-11-12T00:55:01,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/0630ff65950e4c4f9bc151825af1bd32, entries=200, sequenceid=48, filesize=14.0 K 2024-11-12T00:55:01,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/7766fbf116c94964b023c3a83c5ab848 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/7766fbf116c94964b023c3a83c5ab848 2024-11-12T00:55:01,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/7766fbf116c94964b023c3a83c5ab848, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T00:55:01,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/519615f5e9a443abaded6a44fbb8ac1e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/519615f5e9a443abaded6a44fbb8ac1e 2024-11-12T00:55:01,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/519615f5e9a443abaded6a44fbb8ac1e, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T00:55:01,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2400f2d4915c3c25d637e8ed29b4fadd in 1238ms, sequenceid=48, compaction requested=true 2024-11-12T00:55:01,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:01,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:55:01,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:01,387 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:01,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:55:01,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:01,387 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:01,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:55:01,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:01,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-12T00:55:01,389 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:01,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:01,389 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/A is initiating minor compaction (all files) 2024-11-12T00:55:01,389 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/A in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:01,389 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T00:55:01,389 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/18bbb76ed13a43fbb98309ef36ad9422, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/a42839bd7b724c82ba1fc9eb19487bd1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/0630ff65950e4c4f9bc151825af1bd32] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=39.7 K 2024-11-12T00:55:01,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:01,389 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:01,389 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/B is initiating minor compaction (all files) 2024-11-12T00:55:01,389 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/B in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:01,389 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a09c4b85321246bf85ec9c416cf61474, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/85000c30178e49b3895e7af38e559f82, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/7766fbf116c94964b023c3a83c5ab848] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=35.2 K 2024-11-12T00:55:01,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:01,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:01,390 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18bbb76ed13a43fbb98309ef36ad9422, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731372898629 2024-11-12T00:55:01,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:01,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:01,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:01,390 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting a42839bd7b724c82ba1fc9eb19487bd1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731372898660 2024-11-12T00:55:01,390 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting a09c4b85321246bf85ec9c416cf61474, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731372898629 2024-11-12T00:55:01,390 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0630ff65950e4c4f9bc151825af1bd32, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731372899830 2024-11-12T00:55:01,390 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 85000c30178e49b3895e7af38e559f82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731372898660 2024-11-12T00:55:01,391 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 7766fbf116c94964b023c3a83c5ab848, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731372899830 2024-11-12T00:55:01,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/790e2af24aef49aca5c8f5992afdd49f is 50, key is test_row_0/A:col10/1731372900196/Put/seqid=0 2024-11-12T00:55:01,397 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#B#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:01,397 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#A#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:01,398 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/bcd8bd5e3d6c4b20acde20debd86155f is 50, key is test_row_0/B:col10/1731372899830/Put/seqid=0 2024-11-12T00:55:01,398 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/076aca0f9f8943b2a9421a6a4535c45e is 50, key is test_row_0/A:col10/1731372899830/Put/seqid=0 2024-11-12T00:55:01,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742275_1451 (size=12001) 2024-11-12T00:55:01,399 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/790e2af24aef49aca5c8f5992afdd49f 2024-11-12T00:55:01,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742277_1453 (size=12104) 2024-11-12T00:55:01,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742276_1452 (size=12104) 2024-11-12T00:55:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T00:55:01,405 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/076aca0f9f8943b2a9421a6a4535c45e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/076aca0f9f8943b2a9421a6a4535c45e 2024-11-12T00:55:01,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/15c4f1a07a5a4830a96d8ee5605b2ffa is 50, key is test_row_0/B:col10/1731372900196/Put/seqid=0 2024-11-12T00:55:01,409 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/A of 2400f2d4915c3c25d637e8ed29b4fadd into 076aca0f9f8943b2a9421a6a4535c45e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:01,409 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:01,409 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/A, priority=13, startTime=1731372901387; duration=0sec 2024-11-12T00:55:01,409 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:01,409 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:A 2024-11-12T00:55:01,409 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:01,410 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:01,410 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/C is initiating minor compaction (all files) 2024-11-12T00:55:01,410 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/C in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:01,410 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/1ce125510b234f38afd958b71bb2723e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/6a8d31e140554b39a6b1556fb682ee87, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/519615f5e9a443abaded6a44fbb8ac1e] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=35.2 K 2024-11-12T00:55:01,410 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ce125510b234f38afd958b71bb2723e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731372898629 2024-11-12T00:55:01,411 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a8d31e140554b39a6b1556fb682ee87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731372898660 2024-11-12T00:55:01,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742278_1454 (size=12001) 2024-11-12T00:55:01,411 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 519615f5e9a443abaded6a44fbb8ac1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731372899830 2024-11-12T00:55:01,413 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/15c4f1a07a5a4830a96d8ee5605b2ffa 2024-11-12T00:55:01,417 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#C#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:01,417 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/5be8562ac1f14a7db4fcaa069df1212e is 50, key is test_row_0/C:col10/1731372899830/Put/seqid=0 2024-11-12T00:55:01,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/eb0fe16cfe8f4cc2a1e4c753fef0ce24 is 50, key is test_row_0/C:col10/1731372900196/Put/seqid=0 2024-11-12T00:55:01,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742279_1455 (size=12104) 2024-11-12T00:55:01,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742280_1456 (size=12001) 2024-11-12T00:55:01,432 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/eb0fe16cfe8f4cc2a1e4c753fef0ce24 2024-11-12T00:55:01,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/790e2af24aef49aca5c8f5992afdd49f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/790e2af24aef49aca5c8f5992afdd49f 2024-11-12T00:55:01,445 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/790e2af24aef49aca5c8f5992afdd49f, entries=150, sequenceid=65, filesize=11.7 K 2024-11-12T00:55:01,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/15c4f1a07a5a4830a96d8ee5605b2ffa as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/15c4f1a07a5a4830a96d8ee5605b2ffa 2024-11-12T00:55:01,448 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/15c4f1a07a5a4830a96d8ee5605b2ffa, entries=150, sequenceid=65, filesize=11.7 K 2024-11-12T00:55:01,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/eb0fe16cfe8f4cc2a1e4c753fef0ce24 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/eb0fe16cfe8f4cc2a1e4c753fef0ce24 2024-11-12T00:55:01,452 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/eb0fe16cfe8f4cc2a1e4c753fef0ce24, entries=150, sequenceid=65, filesize=11.7 K 2024-11-12T00:55:01,453 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 2400f2d4915c3c25d637e8ed29b4fadd in 63ms, sequenceid=65, compaction requested=false 2024-11-12T00:55:01,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:01,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:01,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-12T00:55:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-12T00:55:01,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-12T00:55:01,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 676 msec 2024-11-12T00:55:01,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 680 msec 2024-11-12T00:55:01,525 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T00:55:01,806 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/bcd8bd5e3d6c4b20acde20debd86155f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/bcd8bd5e3d6c4b20acde20debd86155f 2024-11-12T00:55:01,810 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/B of 2400f2d4915c3c25d637e8ed29b4fadd into bcd8bd5e3d6c4b20acde20debd86155f(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:01,810 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:01,810 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/B, priority=13, startTime=1731372901387; duration=0sec 2024-11-12T00:55:01,810 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:01,810 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:B 2024-11-12T00:55:01,827 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/5be8562ac1f14a7db4fcaa069df1212e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/5be8562ac1f14a7db4fcaa069df1212e 2024-11-12T00:55:01,830 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/C of 2400f2d4915c3c25d637e8ed29b4fadd into 5be8562ac1f14a7db4fcaa069df1212e(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:01,830 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:01,831 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/C, priority=13, startTime=1731372901387; duration=0sec 2024-11-12T00:55:01,831 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:01,831 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:C 2024-11-12T00:55:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T00:55:01,915 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:01,916 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:55:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-12T00:55:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T00:55:01,917 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:55:01,918 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:55:01,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:55:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T00:55:02,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-12T00:55:02,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:02,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:02,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:02,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-12T00:55:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-12T00:55:02,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-12T00:55:02,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-12T00:55:02,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 157 msec 2024-11-12T00:55:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T00:55:02,236 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:02,236 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:55:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-12T00:55:02,238 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:55:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T00:55:02,238 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:55:02,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:55:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T00:55:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:02,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T00:55:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:02,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:02,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/8b93ec99eb4849bb86157deb862aa0d0 is 50, key is test_row_0/A:col10/1731372902372/Put/seqid=0 2024-11-12T00:55:02,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742281_1457 (size=14341) 2024-11-12T00:55:02,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:02,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:02,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:02,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:02,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:02,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:02,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731372902407, max timestamp=1731372902425Num uniques 859; 2024-11-12T00:55:02,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T00:55:02,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731372902407, max timestamp=1731372902425Num uniques 859; 2024-11-12T00:55:02,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T00:55:02,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731372902407, max timestamp=1731372902425Num uniques 859; 2024-11-12T00:55:02,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T00:55:02,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:02,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372962459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:02,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:02,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372962460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:02,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:02,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:02,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372962470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:02,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:02,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372962470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:02,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:02,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:02,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:02,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:02,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:02,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:02,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T00:55:02,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:02,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372962575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:02,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:02,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372962585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:02,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372962585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:02,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372962585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:02,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:02,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:02,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:02,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:02,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:02,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:02,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/8b93ec99eb4849bb86157deb862aa0d0 2024-11-12T00:55:02,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372962785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:02,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/3e66a0fca0164105809bbef613b9c671 is 50, key is test_row_0/B:col10/1731372902372/Put/seqid=0 2024-11-12T00:55:02,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742282_1458 (size=12001) 2024-11-12T00:55:02,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372962795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:02,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372962795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:02,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372962795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:02,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:02,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:02,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:02,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:02,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:02,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T00:55:02,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39678 deadline: 1731372962875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:02,885 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 2400f2d4915c3c25d637e8ed29b4fadd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:02,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:02,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:03,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:03,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:03,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372963096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:03,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372963115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372963115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:03,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372963115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:03,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:03,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:03,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/3e66a0fca0164105809bbef613b9c671 2024-11-12T00:55:03,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/4aa6f956b81d4988816da11fbcebd0d6 is 50, key is test_row_0/C:col10/1731372902372/Put/seqid=0 2024-11-12T00:55:03,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742283_1459 (size=12001) 2024-11-12T00:55:03,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:03,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:03,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T00:55:03,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:03,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:03,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/4aa6f956b81d4988816da11fbcebd0d6 2024-11-12T00:55:03,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/8b93ec99eb4849bb86157deb862aa0d0 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8b93ec99eb4849bb86157deb862aa0d0 2024-11-12T00:55:03,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:03,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:03,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372963605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:03,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8b93ec99eb4849bb86157deb862aa0d0, entries=200, sequenceid=79, filesize=14.0 K 2024-11-12T00:55:03,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/3e66a0fca0164105809bbef613b9c671 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/3e66a0fca0164105809bbef613b9c671 2024-11-12T00:55:03,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/3e66a0fca0164105809bbef613b9c671, entries=150, sequenceid=79, filesize=11.7 K 2024-11-12T00:55:03,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/4aa6f956b81d4988816da11fbcebd0d6 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/4aa6f956b81d4988816da11fbcebd0d6 2024-11-12T00:55:03,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/4aa6f956b81d4988816da11fbcebd0d6, entries=150, sequenceid=79, filesize=11.7 K 2024-11-12T00:55:03,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 2400f2d4915c3c25d637e8ed29b4fadd in 1242ms, sequenceid=79, compaction requested=true 2024-11-12T00:55:03,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:03,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:55:03,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:03,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:55:03,621 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:03,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:03,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:55:03,621 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:03,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:03,621 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:03,621 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:03,621 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/B is initiating minor compaction (all files) 2024-11-12T00:55:03,621 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/A is initiating minor compaction (all files) 2024-11-12T00:55:03,622 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/B in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,622 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/A in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,622 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/bcd8bd5e3d6c4b20acde20debd86155f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/15c4f1a07a5a4830a96d8ee5605b2ffa, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/3e66a0fca0164105809bbef613b9c671] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=35.3 K 2024-11-12T00:55:03,622 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/076aca0f9f8943b2a9421a6a4535c45e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/790e2af24aef49aca5c8f5992afdd49f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8b93ec99eb4849bb86157deb862aa0d0] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=37.5 K 2024-11-12T00:55:03,622 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting bcd8bd5e3d6c4b20acde20debd86155f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731372899830 2024-11-12T00:55:03,622 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 076aca0f9f8943b2a9421a6a4535c45e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731372899830 2024-11-12T00:55:03,622 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c4f1a07a5a4830a96d8ee5605b2ffa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731372900175 2024-11-12T00:55:03,622 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 790e2af24aef49aca5c8f5992afdd49f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731372900175 2024-11-12T00:55:03,622 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e66a0fca0164105809bbef613b9c671, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731372902372 2024-11-12T00:55:03,622 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b93ec99eb4849bb86157deb862aa0d0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731372902367 2024-11-12T00:55:03,630 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#B#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:03,630 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/e5013f93a72b40468b093677f07b6f67 is 50, key is test_row_0/B:col10/1731372902372/Put/seqid=0 2024-11-12T00:55:03,635 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#A#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:03,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742284_1460 (size=12207) 2024-11-12T00:55:03,636 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/7e5500c917ac4f44a9915c232827a4f8 is 50, key is test_row_0/A:col10/1731372902372/Put/seqid=0 2024-11-12T00:55:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:03,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-12T00:55:03,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:03,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:03,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:03,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:03,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:03,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:03,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742285_1461 (size=12207) 2024-11-12T00:55:03,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/1719eaa661314122a61adce2b1b607d7 is 50, key is test_row_0/A:col10/1731372902469/Put/seqid=0 2024-11-12T00:55:03,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742286_1462 (size=14341) 2024-11-12T00:55:03,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372963652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:03,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372963655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:03,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372963655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:03,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:03,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:03,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372963766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372963766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372963766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:03,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:03,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:03,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:03,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:03,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:03,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372963976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:03,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372963985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372963985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:04,039 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/e5013f93a72b40468b093677f07b6f67 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e5013f93a72b40468b093677f07b6f67 2024-11-12T00:55:04,042 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/B of 2400f2d4915c3c25d637e8ed29b4fadd into e5013f93a72b40468b093677f07b6f67(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:04,042 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:04,042 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/B, priority=13, startTime=1731372903621; duration=0sec 2024-11-12T00:55:04,042 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:04,042 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:B 2024-11-12T00:55:04,042 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:04,043 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:04,043 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/C is initiating minor compaction (all files) 2024-11-12T00:55:04,044 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/C in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:04,044 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/5be8562ac1f14a7db4fcaa069df1212e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/eb0fe16cfe8f4cc2a1e4c753fef0ce24, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/4aa6f956b81d4988816da11fbcebd0d6] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=35.3 K 2024-11-12T00:55:04,044 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 5be8562ac1f14a7db4fcaa069df1212e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731372899830 2024-11-12T00:55:04,044 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/7e5500c917ac4f44a9915c232827a4f8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/7e5500c917ac4f44a9915c232827a4f8 2024-11-12T00:55:04,044 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting eb0fe16cfe8f4cc2a1e4c753fef0ce24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731372900175 2024-11-12T00:55:04,045 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aa6f956b81d4988816da11fbcebd0d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731372902372 2024-11-12T00:55:04,048 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/A of 2400f2d4915c3c25d637e8ed29b4fadd into 7e5500c917ac4f44a9915c232827a4f8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:04,048 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:04,048 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/A, priority=13, startTime=1731372903621; duration=0sec 2024-11-12T00:55:04,048 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:04,048 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:A 2024-11-12T00:55:04,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/1719eaa661314122a61adce2b1b607d7 2024-11-12T00:55:04,051 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#C#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:04,051 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/84b7cde577d048dfaee5509291c5b1db is 50, key is test_row_0/C:col10/1731372902372/Put/seqid=0 2024-11-12T00:55:04,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742287_1463 (size=12207) 2024-11-12T00:55:04,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/f129a8b4545e4c57b7f8883428b1174b is 50, key is test_row_0/B:col10/1731372902469/Put/seqid=0 2024-11-12T00:55:04,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742288_1464 (size=12001) 2024-11-12T00:55:04,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:04,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:04,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:04,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:04,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:04,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:04,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:04,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:04,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372964285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:04,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372964295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:04,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:04,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:04,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:04,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372964296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:04,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:04,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:04,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:04,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:04,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:04,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:04,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:04,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T00:55:04,458 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/84b7cde577d048dfaee5509291c5b1db as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/84b7cde577d048dfaee5509291c5b1db 2024-11-12T00:55:04,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/f129a8b4545e4c57b7f8883428b1174b 2024-11-12T00:55:04,462 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/C of 2400f2d4915c3c25d637e8ed29b4fadd into 84b7cde577d048dfaee5509291c5b1db(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:04,462 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:04,462 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/C, priority=13, startTime=1731372903621; duration=0sec 2024-11-12T00:55:04,462 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:04,462 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:C 2024-11-12T00:55:04,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/f279f6f236d24fb9b220807f93beafc4 is 50, key is test_row_0/C:col10/1731372902469/Put/seqid=0 2024-11-12T00:55:04,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742289_1465 (size=12001) 2024-11-12T00:55:04,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:04,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:04,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:04,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:04,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372964625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:04,633 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 2400f2d4915c3c25d637e8ed29b4fadd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:04,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:04,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:04,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:04,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:04,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:04,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372964806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:04,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:04,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372964815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:04,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:04,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372964815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:04,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:04,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:04,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:04,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/f279f6f236d24fb9b220807f93beafc4 2024-11-12T00:55:04,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/1719eaa661314122a61adce2b1b607d7 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/1719eaa661314122a61adce2b1b607d7 2024-11-12T00:55:04,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/1719eaa661314122a61adce2b1b607d7, entries=200, sequenceid=116, filesize=14.0 K 2024-11-12T00:55:04,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/f129a8b4545e4c57b7f8883428b1174b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/f129a8b4545e4c57b7f8883428b1174b 2024-11-12T00:55:04,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/f129a8b4545e4c57b7f8883428b1174b, entries=150, sequenceid=116, filesize=11.7 K 2024-11-12T00:55:04,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/f279f6f236d24fb9b220807f93beafc4 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/f279f6f236d24fb9b220807f93beafc4 2024-11-12T00:55:04,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/f279f6f236d24fb9b220807f93beafc4, entries=150, sequenceid=116, filesize=11.7 K 2024-11-12T00:55:04,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 2400f2d4915c3c25d637e8ed29b4fadd in 1240ms, sequenceid=116, compaction requested=false 2024-11-12T00:55:04,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:04,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T00:55:04,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:04,986 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T00:55:04,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:04,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:04,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:04,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:04,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:04,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:04,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/760ce223f7e540c2ab2233b26f8b60da is 50, key is test_row_0/A:col10/1731372903640/Put/seqid=0 2024-11-12T00:55:04,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742290_1466 (size=12001) 2024-11-12T00:55:04,998 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/760ce223f7e540c2ab2233b26f8b60da 2024-11-12T00:55:05,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/56c3662b53f246b09d94dea9a4caca11 is 50, key is test_row_0/B:col10/1731372903640/Put/seqid=0 2024-11-12T00:55:05,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742291_1467 (size=12001) 2024-11-12T00:55:05,410 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/56c3662b53f246b09d94dea9a4caca11 2024-11-12T00:55:05,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/6cacdc7c7d014b19998c6ba06f795411 is 50, key is test_row_0/C:col10/1731372903640/Put/seqid=0 2024-11-12T00:55:05,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742292_1468 (size=12001) 2024-11-12T00:55:05,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:05,845 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/6cacdc7c7d014b19998c6ba06f795411 2024-11-12T00:55:05,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/760ce223f7e540c2ab2233b26f8b60da as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/760ce223f7e540c2ab2233b26f8b60da 2024-11-12T00:55:05,852 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/760ce223f7e540c2ab2233b26f8b60da, entries=150, sequenceid=127, filesize=11.7 K 2024-11-12T00:55:05,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/56c3662b53f246b09d94dea9a4caca11 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/56c3662b53f246b09d94dea9a4caca11 2024-11-12T00:55:05,855 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/56c3662b53f246b09d94dea9a4caca11, entries=150, sequenceid=127, filesize=11.7 K 2024-11-12T00:55:05,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/6cacdc7c7d014b19998c6ba06f795411 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/6cacdc7c7d014b19998c6ba06f795411 2024-11-12T00:55:05,859 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/6cacdc7c7d014b19998c6ba06f795411, entries=150, sequenceid=127, filesize=11.7 K 2024-11-12T00:55:05,860 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=80.51 KB/82440 for 2400f2d4915c3c25d637e8ed29b4fadd in 874ms, sequenceid=127, compaction requested=true 2024-11-12T00:55:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-12T00:55:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-12T00:55:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:05,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T00:55:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:05,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-12T00:55:05,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6230 sec 2024-11-12T00:55:05,865 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 3.6270 sec 2024-11-12T00:55:05,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/57ec1a4041984adcbe7503ff59d75f10 is 50, key is test_row_0/A:col10/1731372905859/Put/seqid=0 2024-11-12T00:55:05,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742293_1469 (size=19321) 2024-11-12T00:55:05,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/57ec1a4041984adcbe7503ff59d75f10 2024-11-12T00:55:05,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/a740b10ddd9049d6bfe8f5cd975a1bd5 is 50, key is test_row_0/B:col10/1731372905859/Put/seqid=0 2024-11-12T00:55:05,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742294_1470 (size=12151) 2024-11-12T00:55:05,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/a740b10ddd9049d6bfe8f5cd975a1bd5 2024-11-12T00:55:05,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/bbe2436b2aa44b979bebe66de633f228 is 50, key is test_row_0/C:col10/1731372905859/Put/seqid=0 2024-11-12T00:55:05,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742295_1471 (size=12151) 2024-11-12T00:55:05,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372965898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:05,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372965899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:05,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372965909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372966016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372966016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372966025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372966225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372966225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372966236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/bbe2436b2aa44b979bebe66de633f228 2024-11-12T00:55:06,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/57ec1a4041984adcbe7503ff59d75f10 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/57ec1a4041984adcbe7503ff59d75f10 2024-11-12T00:55:06,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/57ec1a4041984adcbe7503ff59d75f10, entries=300, sequenceid=143, filesize=18.9 K 2024-11-12T00:55:06,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/a740b10ddd9049d6bfe8f5cd975a1bd5 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a740b10ddd9049d6bfe8f5cd975a1bd5 2024-11-12T00:55:06,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a740b10ddd9049d6bfe8f5cd975a1bd5, entries=150, sequenceid=143, filesize=11.9 K 2024-11-12T00:55:06,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/bbe2436b2aa44b979bebe66de633f228 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/bbe2436b2aa44b979bebe66de633f228 2024-11-12T00:55:06,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/bbe2436b2aa44b979bebe66de633f228, entries=150, sequenceid=143, filesize=11.9 K 2024-11-12T00:55:06,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2400f2d4915c3c25d637e8ed29b4fadd in 453ms, sequenceid=143, compaction requested=true 2024-11-12T00:55:06,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:06,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:55:06,315 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:55:06,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:06,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:55:06,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:06,315 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:55:06,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:55:06,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:06,315 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:55:06,315 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57870 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:55:06,315 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/B is initiating minor compaction (all files) 2024-11-12T00:55:06,315 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/A is initiating minor compaction (all files) 2024-11-12T00:55:06,316 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/B in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:06,316 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/A in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:06,316 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e5013f93a72b40468b093677f07b6f67, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/f129a8b4545e4c57b7f8883428b1174b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/56c3662b53f246b09d94dea9a4caca11, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a740b10ddd9049d6bfe8f5cd975a1bd5] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=47.2 K 2024-11-12T00:55:06,316 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/7e5500c917ac4f44a9915c232827a4f8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/1719eaa661314122a61adce2b1b607d7, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/760ce223f7e540c2ab2233b26f8b60da, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/57ec1a4041984adcbe7503ff59d75f10] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=56.5 K 2024-11-12T00:55:06,316 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting e5013f93a72b40468b093677f07b6f67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731372902372 2024-11-12T00:55:06,316 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e5500c917ac4f44a9915c232827a4f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731372902372 2024-11-12T00:55:06,316 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting f129a8b4545e4c57b7f8883428b1174b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731372902458 2024-11-12T00:55:06,316 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1719eaa661314122a61adce2b1b607d7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731372902455 2024-11-12T00:55:06,316 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 56c3662b53f246b09d94dea9a4caca11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731372903640 2024-11-12T00:55:06,316 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 760ce223f7e540c2ab2233b26f8b60da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731372903640 2024-11-12T00:55:06,316 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting a740b10ddd9049d6bfe8f5cd975a1bd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731372905856 2024-11-12T00:55:06,316 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57ec1a4041984adcbe7503ff59d75f10, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731372905844 2024-11-12T00:55:06,323 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#B#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:06,324 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/e68a531add0d457cb9bc6f98700e9f01 is 50, key is test_row_0/B:col10/1731372905859/Put/seqid=0 2024-11-12T00:55:06,326 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#A#compaction#400 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:06,326 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/8424ca1d7d044ab5a900b1fcdceef439 is 50, key is test_row_0/A:col10/1731372905859/Put/seqid=0 2024-11-12T00:55:06,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742296_1472 (size=12493) 2024-11-12T00:55:06,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742297_1473 (size=12493) 2024-11-12T00:55:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T00:55:06,396 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:06,396 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:55:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-12T00:55:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T00:55:06,398 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:55:06,399 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:55:06,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:55:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T00:55:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:06,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T00:55:06,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:06,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:06,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:06,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:06,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:06,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:06,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/4f0c9920c00e435fa7c11b009baa4835 is 50, key is test_row_0/A:col10/1731372905898/Put/seqid=0 2024-11-12T00:55:06,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T00:55:06,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:06,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:06,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:06,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:06,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:06,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742298_1474 (size=14541) 2024-11-12T00:55:06,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/4f0c9920c00e435fa7c11b009baa4835 2024-11-12T00:55:06,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/b08b4b8210c94b52bb6d2ab953959413 is 50, key is test_row_0/B:col10/1731372905898/Put/seqid=0 2024-11-12T00:55:06,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742299_1475 (size=12151) 2024-11-12T00:55:06,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372966586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372966598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372966599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372966655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,665 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 2400f2d4915c3c25d637e8ed29b4fadd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:06,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T00:55:06,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:06,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:06,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:06,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:06,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:06,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372966705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372966705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372966705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T00:55:06,739 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/e68a531add0d457cb9bc6f98700e9f01 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e68a531add0d457cb9bc6f98700e9f01 2024-11-12T00:55:06,742 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/B of 2400f2d4915c3c25d637e8ed29b4fadd into e68a531add0d457cb9bc6f98700e9f01(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:06,742 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:06,742 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/B, priority=12, startTime=1731372906315; duration=0sec 2024-11-12T00:55:06,742 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:06,742 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:B 2024-11-12T00:55:06,742 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:55:06,743 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:55:06,743 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/C is initiating minor compaction (all files) 2024-11-12T00:55:06,743 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/C in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:06,743 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/84b7cde577d048dfaee5509291c5b1db, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/f279f6f236d24fb9b220807f93beafc4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/6cacdc7c7d014b19998c6ba06f795411, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/bbe2436b2aa44b979bebe66de633f228] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=47.2 K 2024-11-12T00:55:06,744 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 84b7cde577d048dfaee5509291c5b1db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731372902372 2024-11-12T00:55:06,744 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting f279f6f236d24fb9b220807f93beafc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731372902458 2024-11-12T00:55:06,744 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cacdc7c7d014b19998c6ba06f795411, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731372903640 2024-11-12T00:55:06,745 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting bbe2436b2aa44b979bebe66de633f228, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731372905856 2024-11-12T00:55:06,747 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/8424ca1d7d044ab5a900b1fcdceef439 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8424ca1d7d044ab5a900b1fcdceef439 2024-11-12T00:55:06,752 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/A of 2400f2d4915c3c25d637e8ed29b4fadd into 8424ca1d7d044ab5a900b1fcdceef439(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:06,752 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:06,752 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/A, priority=12, startTime=1731372906315; duration=0sec 2024-11-12T00:55:06,752 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:06,752 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:A 2024-11-12T00:55:06,753 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#C#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:06,753 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/91481ab0f2de4059a732d84dc50009d2 is 50, key is test_row_0/C:col10/1731372905859/Put/seqid=0 2024-11-12T00:55:06,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742300_1476 (size=12493) 2024-11-12T00:55:06,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T00:55:06,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:06,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:06,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:06,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:06,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:06,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39678 deadline: 1731372966915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372966915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,924 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 2400f2d4915c3c25d637e8ed29b4fadd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:06,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372966915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372966915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:06,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/b08b4b8210c94b52bb6d2ab953959413 2024-11-12T00:55:06,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/7e9f1f40084a4ac2a2f4fbfaffb1e3ce is 50, key is test_row_0/C:col10/1731372905898/Put/seqid=0 2024-11-12T00:55:06,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742301_1477 (size=12151) 2024-11-12T00:55:07,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T00:55:07,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:07,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:07,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:07,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:07,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T00:55:07,160 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/91481ab0f2de4059a732d84dc50009d2 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/91481ab0f2de4059a732d84dc50009d2 2024-11-12T00:55:07,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T00:55:07,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:07,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:07,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:07,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:07,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:07,165 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/C of 2400f2d4915c3c25d637e8ed29b4fadd into 91481ab0f2de4059a732d84dc50009d2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:07,166 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:07,166 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/C, priority=12, startTime=1731372906315; duration=0sec 2024-11-12T00:55:07,166 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:07,166 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:C 2024-11-12T00:55:07,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372967236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372967236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:07,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372967236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:07,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T00:55:07,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:07,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:07,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:07,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:07,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:07,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/7e9f1f40084a4ac2a2f4fbfaffb1e3ce 2024-11-12T00:55:07,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/4f0c9920c00e435fa7c11b009baa4835 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/4f0c9920c00e435fa7c11b009baa4835 2024-11-12T00:55:07,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/4f0c9920c00e435fa7c11b009baa4835, entries=200, sequenceid=160, filesize=14.2 K 2024-11-12T00:55:07,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/b08b4b8210c94b52bb6d2ab953959413 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b08b4b8210c94b52bb6d2ab953959413 2024-11-12T00:55:07,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b08b4b8210c94b52bb6d2ab953959413, entries=150, sequenceid=160, filesize=11.9 K 2024-11-12T00:55:07,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/7e9f1f40084a4ac2a2f4fbfaffb1e3ce as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/7e9f1f40084a4ac2a2f4fbfaffb1e3ce 2024-11-12T00:55:07,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/7e9f1f40084a4ac2a2f4fbfaffb1e3ce, entries=150, sequenceid=160, filesize=11.9 K 2024-11-12T00:55:07,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 2400f2d4915c3c25d637e8ed29b4fadd in 843ms, sequenceid=160, compaction requested=false 2024-11-12T00:55:07,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:07,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T00:55:07,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:07,468 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T00:55:07,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:07,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:07,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:07,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:07,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:07,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:07,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/da0d98de76c243dbb835b157bdd2b7dd is 50, key is test_row_0/A:col10/1731372906583/Put/seqid=0 2024-11-12T00:55:07,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742302_1478 (size=12151) 2024-11-12T00:55:07,476 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/da0d98de76c243dbb835b157bdd2b7dd 2024-11-12T00:55:07,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/b2aa834ce30c40ccbb35ca285db51044 is 50, key is test_row_0/B:col10/1731372906583/Put/seqid=0 2024-11-12T00:55:07,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742303_1479 (size=12151) 2024-11-12T00:55:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T00:55:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:07,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:07,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372967788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:07,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:07,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372967789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:07,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372967791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:07,887 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/b2aa834ce30c40ccbb35ca285db51044 2024-11-12T00:55:07,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/aeeb4175d70d4cbeb56162be2df1624d is 50, key is test_row_0/C:col10/1731372906583/Put/seqid=0 2024-11-12T00:55:07,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742304_1480 (size=12151) 2024-11-12T00:55:07,896 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/aeeb4175d70d4cbeb56162be2df1624d 2024-11-12T00:55:07,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:07,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372967896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:07,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/da0d98de76c243dbb835b157bdd2b7dd as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/da0d98de76c243dbb835b157bdd2b7dd 2024-11-12T00:55:07,902 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/da0d98de76c243dbb835b157bdd2b7dd, entries=150, sequenceid=178, filesize=11.9 K 2024-11-12T00:55:07,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/b2aa834ce30c40ccbb35ca285db51044 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b2aa834ce30c40ccbb35ca285db51044 2024-11-12T00:55:07,906 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b2aa834ce30c40ccbb35ca285db51044, entries=150, sequenceid=178, filesize=11.9 K 2024-11-12T00:55:07,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/aeeb4175d70d4cbeb56162be2df1624d as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/aeeb4175d70d4cbeb56162be2df1624d 2024-11-12T00:55:07,910 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/aeeb4175d70d4cbeb56162be2df1624d, entries=150, sequenceid=178, filesize=11.9 K 2024-11-12T00:55:07,911 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2400f2d4915c3c25d637e8ed29b4fadd in 443ms, sequenceid=178, compaction requested=true 2024-11-12T00:55:07,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:07,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:07,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-12T00:55:07,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-12T00:55:07,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:07,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T00:55:07,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:07,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:07,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:07,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:07,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:07,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:07,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-12T00:55:07,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5130 sec 2024-11-12T00:55:07,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.5180 sec 2024-11-12T00:55:07,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/8487a74c12b74757b1e8a3aff62e4098 is 50, key is test_row_0/A:col10/1731372907777/Put/seqid=0 2024-11-12T00:55:07,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742305_1481 (size=14541) 2024-11-12T00:55:07,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/8487a74c12b74757b1e8a3aff62e4098 2024-11-12T00:55:07,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/7aa701a151ee499283d210571ce7243d is 50, key is test_row_0/B:col10/1731372907777/Put/seqid=0 2024-11-12T00:55:07,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742306_1482 (size=12151) 2024-11-12T00:55:07,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:07,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372967972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:07,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372967973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:07,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:08,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372968075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:08,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372968085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:08,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372968106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:08,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372968285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:08,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372968295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:08,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/7aa701a151ee499283d210571ce7243d 2024-11-12T00:55:08,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/c3c06a1cb8c64632abcaa11933793ce2 is 50, key is test_row_0/C:col10/1731372907777/Put/seqid=0 2024-11-12T00:55:08,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742307_1483 (size=12151) 2024-11-12T00:55:08,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372968415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T00:55:08,556 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:08,556 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:55:08,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-12T00:55:08,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T00:55:08,557 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:55:08,558 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:55:08,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:55:08,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:08,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372968596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:08,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372968605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T00:55:08,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T00:55:08,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:08,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:08,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:08,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:08,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:08,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/c3c06a1cb8c64632abcaa11933793ce2 2024-11-12T00:55:08,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/8487a74c12b74757b1e8a3aff62e4098 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8487a74c12b74757b1e8a3aff62e4098 2024-11-12T00:55:08,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8487a74c12b74757b1e8a3aff62e4098, entries=200, sequenceid=196, filesize=14.2 K 2024-11-12T00:55:08,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/7aa701a151ee499283d210571ce7243d as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/7aa701a151ee499283d210571ce7243d 2024-11-12T00:55:08,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/7aa701a151ee499283d210571ce7243d, entries=150, sequenceid=196, filesize=11.9 K 2024-11-12T00:55:08,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/c3c06a1cb8c64632abcaa11933793ce2 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/c3c06a1cb8c64632abcaa11933793ce2 2024-11-12T00:55:08,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/c3c06a1cb8c64632abcaa11933793ce2, entries=150, sequenceid=196, filesize=11.9 K 2024-11-12T00:55:08,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 2400f2d4915c3c25d637e8ed29b4fadd in 842ms, sequenceid=196, compaction requested=true 2024-11-12T00:55:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:55:08,755 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:55:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:55:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:08,755 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:55:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:55:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:08,756 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:55:08,756 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:55:08,756 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/A is initiating minor compaction (all files) 2024-11-12T00:55:08,756 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/B is initiating minor compaction (all files) 2024-11-12T00:55:08,756 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/B in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:08,756 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/A in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:08,756 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8424ca1d7d044ab5a900b1fcdceef439, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/4f0c9920c00e435fa7c11b009baa4835, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/da0d98de76c243dbb835b157bdd2b7dd, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8487a74c12b74757b1e8a3aff62e4098] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=52.5 K 2024-11-12T00:55:08,756 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e68a531add0d457cb9bc6f98700e9f01, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b08b4b8210c94b52bb6d2ab953959413, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b2aa834ce30c40ccbb35ca285db51044, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/7aa701a151ee499283d210571ce7243d] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=47.8 K 2024-11-12T00:55:08,756 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8424ca1d7d044ab5a900b1fcdceef439, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731372905856 2024-11-12T00:55:08,756 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting e68a531add0d457cb9bc6f98700e9f01, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731372905856 2024-11-12T00:55:08,756 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting b08b4b8210c94b52bb6d2ab953959413, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731372905876 2024-11-12T00:55:08,756 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f0c9920c00e435fa7c11b009baa4835, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731372905876 2024-11-12T00:55:08,757 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting b2aa834ce30c40ccbb35ca285db51044, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731372906583 2024-11-12T00:55:08,757 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting da0d98de76c243dbb835b157bdd2b7dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731372906583 2024-11-12T00:55:08,757 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8487a74c12b74757b1e8a3aff62e4098, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731372907777 2024-11-12T00:55:08,757 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aa701a151ee499283d210571ce7243d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731372907777 2024-11-12T00:55:08,767 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#B#compaction#411 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:08,767 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/0d9368d304e34c3d97fd90c7d5bb69ab is 50, key is test_row_0/B:col10/1731372907777/Put/seqid=0 2024-11-12T00:55:08,768 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#A#compaction#412 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:08,768 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/f0feda69f8734b7abd2f4d4403186407 is 50, key is test_row_0/A:col10/1731372907777/Put/seqid=0 2024-11-12T00:55:08,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742308_1484 (size=12629) 2024-11-12T00:55:08,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742309_1485 (size=12629) 2024-11-12T00:55:08,786 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/f0feda69f8734b7abd2f4d4403186407 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/f0feda69f8734b7abd2f4d4403186407 2024-11-12T00:55:08,790 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/A of 2400f2d4915c3c25d637e8ed29b4fadd into f0feda69f8734b7abd2f4d4403186407(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:08,791 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:08,791 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/A, priority=12, startTime=1731372908755; duration=0sec 2024-11-12T00:55:08,791 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:08,791 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:A 2024-11-12T00:55:08,791 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:55:08,792 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:55:08,792 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/C is initiating minor compaction (all files) 2024-11-12T00:55:08,792 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/C in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:08,792 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/91481ab0f2de4059a732d84dc50009d2, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/7e9f1f40084a4ac2a2f4fbfaffb1e3ce, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/aeeb4175d70d4cbeb56162be2df1624d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/c3c06a1cb8c64632abcaa11933793ce2] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=47.8 K 2024-11-12T00:55:08,792 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91481ab0f2de4059a732d84dc50009d2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731372905856 2024-11-12T00:55:08,792 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e9f1f40084a4ac2a2f4fbfaffb1e3ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731372905876 2024-11-12T00:55:08,793 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeeb4175d70d4cbeb56162be2df1624d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731372906583 2024-11-12T00:55:08,793 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3c06a1cb8c64632abcaa11933793ce2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731372907777 2024-11-12T00:55:08,799 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#C#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:08,800 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/8b65f7a267574d55b2469761229a9c7b is 50, key is test_row_0/C:col10/1731372907777/Put/seqid=0 2024-11-12T00:55:08,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742310_1486 (size=12629) 2024-11-12T00:55:08,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T00:55:08,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:08,863 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T00:55:08,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:08,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:08,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:08,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:08,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:08,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:08,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/ba6e6b1624e14967a0ecf7ca694c36a9 is 50, key is test_row_0/A:col10/1731372907960/Put/seqid=0 2024-11-12T00:55:08,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742311_1487 (size=12151) 2024-11-12T00:55:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T00:55:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:08,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:09,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372969014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:09,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372969115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:09,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:09,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:09,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:09,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372969115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:09,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372969125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:09,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:09,180 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/0d9368d304e34c3d97fd90c7d5bb69ab as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/0d9368d304e34c3d97fd90c7d5bb69ab 2024-11-12T00:55:09,183 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/B of 2400f2d4915c3c25d637e8ed29b4fadd into 0d9368d304e34c3d97fd90c7d5bb69ab(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:09,183 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:09,183 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/B, priority=12, startTime=1731372908755; duration=0sec 2024-11-12T00:55:09,183 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:09,183 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:B 2024-11-12T00:55:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T00:55:09,208 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/8b65f7a267574d55b2469761229a9c7b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/8b65f7a267574d55b2469761229a9c7b 2024-11-12T00:55:09,211 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/C of 2400f2d4915c3c25d637e8ed29b4fadd into 8b65f7a267574d55b2469761229a9c7b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:09,211 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:09,211 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/C, priority=12, startTime=1731372908755; duration=0sec 2024-11-12T00:55:09,212 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:09,212 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:C 2024-11-12T00:55:09,272 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/ba6e6b1624e14967a0ecf7ca694c36a9 2024-11-12T00:55:09,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/f1ef93a1f01f485bab459ff06f971c0b is 50, key is test_row_0/B:col10/1731372907960/Put/seqid=0 2024-11-12T00:55:09,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742312_1488 (size=12151) 2024-11-12T00:55:09,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372969345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:09,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372969665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:09,680 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/f1ef93a1f01f485bab459ff06f971c0b 2024-11-12T00:55:09,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/0d615d75296f4b88b9ea2d8bfc6faea6 is 50, key is test_row_0/C:col10/1731372907960/Put/seqid=0 2024-11-12T00:55:09,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742313_1489 (size=12151) 2024-11-12T00:55:09,691 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/0d615d75296f4b88b9ea2d8bfc6faea6 2024-11-12T00:55:09,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/ba6e6b1624e14967a0ecf7ca694c36a9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/ba6e6b1624e14967a0ecf7ca694c36a9 2024-11-12T00:55:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T00:55:09,697 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/ba6e6b1624e14967a0ecf7ca694c36a9, entries=150, sequenceid=212, filesize=11.9 K 2024-11-12T00:55:09,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/f1ef93a1f01f485bab459ff06f971c0b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/f1ef93a1f01f485bab459ff06f971c0b 2024-11-12T00:55:09,700 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/f1ef93a1f01f485bab459ff06f971c0b, entries=150, sequenceid=212, filesize=11.9 K 2024-11-12T00:55:09,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/0d615d75296f4b88b9ea2d8bfc6faea6 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/0d615d75296f4b88b9ea2d8bfc6faea6 2024-11-12T00:55:09,703 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/0d615d75296f4b88b9ea2d8bfc6faea6, entries=150, sequenceid=212, filesize=11.9 K 2024-11-12T00:55:09,704 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2400f2d4915c3c25d637e8ed29b4fadd in 841ms, sequenceid=212, compaction requested=false 2024-11-12T00:55:09,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:09,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:09,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-12T00:55:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-12T00:55:09,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-12T00:55:09,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1470 sec 2024-11-12T00:55:09,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.1510 sec 2024-11-12T00:55:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:10,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T00:55:10,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:10,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:10,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:10,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:10,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:10,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:10,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/6514a1d11dc343fd9351e068545b002f is 50, key is test_row_0/A:col10/1731372908975/Put/seqid=0 2024-11-12T00:55:10,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742314_1490 (size=14541) 2024-11-12T00:55:10,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372970179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:10,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372970182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:10,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:10,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372970186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:10,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:10,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372970285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:10,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372970296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:10,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372970495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:10,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372970505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:10,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/6514a1d11dc343fd9351e068545b002f 2024-11-12T00:55:10,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/b33a2e8864334d6e8e2af52af4723163 is 50, key is test_row_0/B:col10/1731372908975/Put/seqid=0 2024-11-12T00:55:10,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742315_1491 (size=12151) 2024-11-12T00:55:10,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39630 deadline: 1731372970685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:10,691 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 2400f2d4915c3c25d637e8ed29b4fadd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:10,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T00:55:10,705 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:10,706 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:55:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-12T00:55:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T00:55:10,707 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:55:10,708 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:55:10,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:55:10,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372970805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T00:55:10,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372970825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:10,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T00:55:10,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:10,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:10,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:10,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:10,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:10,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/b33a2e8864334d6e8e2af52af4723163 2024-11-12T00:55:10,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/d82754ba30f346d393f4cd67477c3e43 is 50, key is test_row_0/C:col10/1731372908975/Put/seqid=0 2024-11-12T00:55:10,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742316_1492 (size=12151) 2024-11-12T00:55:11,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T00:55:11,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:11,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:11,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:11,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:11,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T00:55:11,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T00:55:11,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:11,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:11,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:11,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:11,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:11,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372971195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:11,203 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_1' in 2400f2d4915c3c25d637e8ed29b4fadd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:11,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:11,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:11,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:11,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T00:55:11,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:11,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:11,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:11,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:11,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:11,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372971325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T00:55:11,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372971346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:11,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:11,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:11,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:11,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/d82754ba30f346d393f4cd67477c3e43 2024-11-12T00:55:11,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/6514a1d11dc343fd9351e068545b002f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/6514a1d11dc343fd9351e068545b002f 2024-11-12T00:55:11,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/6514a1d11dc343fd9351e068545b002f, entries=200, sequenceid=232, filesize=14.2 K 2024-11-12T00:55:11,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/b33a2e8864334d6e8e2af52af4723163 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b33a2e8864334d6e8e2af52af4723163 2024-11-12T00:55:11,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b33a2e8864334d6e8e2af52af4723163, entries=150, sequenceid=232, filesize=11.9 K 2024-11-12T00:55:11,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/d82754ba30f346d393f4cd67477c3e43 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/d82754ba30f346d393f4cd67477c3e43 2024-11-12T00:55:11,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/d82754ba30f346d393f4cd67477c3e43, entries=150, sequenceid=232, filesize=11.9 K 2024-11-12T00:55:11,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2400f2d4915c3c25d637e8ed29b4fadd in 1239ms, sequenceid=232, compaction requested=true 2024-11-12T00:55:11,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:11,379 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:11,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:55:11,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:11,380 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:11,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:55:11,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:11,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:55:11,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:11,380 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:11,380 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/A is initiating minor compaction (all files) 2024-11-12T00:55:11,380 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/A in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:11,380 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/f0feda69f8734b7abd2f4d4403186407, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/ba6e6b1624e14967a0ecf7ca694c36a9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/6514a1d11dc343fd9351e068545b002f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=38.4 K 2024-11-12T00:55:11,381 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:11,381 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/B is initiating minor compaction (all files) 2024-11-12T00:55:11,381 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0feda69f8734b7abd2f4d4403186407, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731372907777 2024-11-12T00:55:11,381 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/B in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:11,381 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/0d9368d304e34c3d97fd90c7d5bb69ab, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/f1ef93a1f01f485bab459ff06f971c0b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b33a2e8864334d6e8e2af52af4723163] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=36.1 K 2024-11-12T00:55:11,381 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba6e6b1624e14967a0ecf7ca694c36a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731372907940 2024-11-12T00:55:11,381 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d9368d304e34c3d97fd90c7d5bb69ab, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731372907777 2024-11-12T00:55:11,381 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6514a1d11dc343fd9351e068545b002f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731372908975 2024-11-12T00:55:11,382 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting f1ef93a1f01f485bab459ff06f971c0b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731372907940 2024-11-12T00:55:11,382 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting b33a2e8864334d6e8e2af52af4723163, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731372908975 2024-11-12T00:55:11,387 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#A#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:11,388 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/bd6ddd62d48b4a8c87949c3c20048ab9 is 50, key is test_row_0/A:col10/1731372908975/Put/seqid=0 2024-11-12T00:55:11,391 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#B#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:11,391 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/d83f1603ce154d0c8215f37569fd1f9d is 50, key is test_row_0/B:col10/1731372908975/Put/seqid=0 2024-11-12T00:55:11,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742317_1493 (size=12731) 2024-11-12T00:55:11,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742318_1494 (size=12731) 2024-11-12T00:55:11,405 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/d83f1603ce154d0c8215f37569fd1f9d as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/d83f1603ce154d0c8215f37569fd1f9d 2024-11-12T00:55:11,408 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/B of 2400f2d4915c3c25d637e8ed29b4fadd into d83f1603ce154d0c8215f37569fd1f9d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:11,408 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:11,408 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/B, priority=13, startTime=1731372911380; duration=0sec 2024-11-12T00:55:11,408 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:11,408 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:B 2024-11-12T00:55:11,408 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:11,409 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:11,409 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/C is initiating minor compaction (all files) 2024-11-12T00:55:11,409 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/C in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:11,409 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/8b65f7a267574d55b2469761229a9c7b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/0d615d75296f4b88b9ea2d8bfc6faea6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/d82754ba30f346d393f4cd67477c3e43] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=36.1 K 2024-11-12T00:55:11,409 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b65f7a267574d55b2469761229a9c7b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731372907777 2024-11-12T00:55:11,410 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d615d75296f4b88b9ea2d8bfc6faea6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731372907940 2024-11-12T00:55:11,410 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting d82754ba30f346d393f4cd67477c3e43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731372908975 2024-11-12T00:55:11,417 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#C#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:11,418 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/73ef3893f4454e81bbe002ec272c0f26 is 50, key is test_row_0/C:col10/1731372908975/Put/seqid=0 2024-11-12T00:55:11,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742319_1495 (size=12731) 2024-11-12T00:55:11,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T00:55:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:11,477 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T00:55:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:11,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/d5a9622894374a80824708c534169a87 is 50, key is test_row_0/A:col10/1731372910178/Put/seqid=0 2024-11-12T00:55:11,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742320_1496 (size=12151) 2024-11-12T00:55:11,797 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/bd6ddd62d48b4a8c87949c3c20048ab9 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/bd6ddd62d48b4a8c87949c3c20048ab9 2024-11-12T00:55:11,801 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/A of 2400f2d4915c3c25d637e8ed29b4fadd into bd6ddd62d48b4a8c87949c3c20048ab9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:11,801 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:11,801 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/A, priority=13, startTime=1731372911379; duration=0sec 2024-11-12T00:55:11,801 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:11,801 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:A 2024-11-12T00:55:11,824 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/73ef3893f4454e81bbe002ec272c0f26 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/73ef3893f4454e81bbe002ec272c0f26 2024-11-12T00:55:11,827 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/C of 2400f2d4915c3c25d637e8ed29b4fadd into 73ef3893f4454e81bbe002ec272c0f26(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:11,827 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:11,827 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/C, priority=13, startTime=1731372911380; duration=0sec 2024-11-12T00:55:11,828 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:11,828 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:C 2024-11-12T00:55:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T00:55:11,890 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/d5a9622894374a80824708c534169a87 2024-11-12T00:55:11,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/8f1ba4715b7c40a59fe24e7b5340b19b is 50, key is test_row_0/B:col10/1731372910178/Put/seqid=0 2024-11-12T00:55:11,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742321_1497 (size=12151) 2024-11-12T00:55:11,903 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/8f1ba4715b7c40a59fe24e7b5340b19b 2024-11-12T00:55:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/4b6c14ec32fc464ebe850a42fd74ff0c is 50, key is test_row_0/C:col10/1731372910178/Put/seqid=0 2024-11-12T00:55:11,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742322_1498 (size=12151) 2024-11-12T00:55:12,319 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/4b6c14ec32fc464ebe850a42fd74ff0c 2024-11-12T00:55:12,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/d5a9622894374a80824708c534169a87 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/d5a9622894374a80824708c534169a87 2024-11-12T00:55:12,325 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/d5a9622894374a80824708c534169a87, entries=150, sequenceid=247, filesize=11.9 K 2024-11-12T00:55:12,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/8f1ba4715b7c40a59fe24e7b5340b19b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/8f1ba4715b7c40a59fe24e7b5340b19b 2024-11-12T00:55:12,329 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/8f1ba4715b7c40a59fe24e7b5340b19b, entries=150, sequenceid=247, filesize=11.9 K 2024-11-12T00:55:12,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/4b6c14ec32fc464ebe850a42fd74ff0c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/4b6c14ec32fc464ebe850a42fd74ff0c 2024-11-12T00:55:12,332 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/4b6c14ec32fc464ebe850a42fd74ff0c, entries=150, sequenceid=247, filesize=11.9 K 2024-11-12T00:55:12,333 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 2400f2d4915c3c25d637e8ed29b4fadd in 856ms, sequenceid=247, compaction requested=false 2024-11-12T00:55:12,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:12,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:12,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-12T00:55:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-12T00:55:12,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-12T00:55:12,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6260 sec 2024-11-12T00:55:12,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.6300 sec 2024-11-12T00:55:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:12,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T00:55:12,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:12,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:12,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:12,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:12,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:12,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:12,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/8bc6316e3a8542c9b8aeceaef6589bfd is 50, key is test_row_0/A:col10/1731372912361/Put/seqid=0 2024-11-12T00:55:12,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742323_1499 (size=12301) 2024-11-12T00:55:12,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372912397, max timestamp=1731372912447Num uniques 859; 2024-11-12T00:55:12,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372912397, max timestamp=1731372912447Num uniques 859; 2024-11-12T00:55:12,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:55:12,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:55:12,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731372912397, max timestamp=1731372912447Num uniques 859; 2024-11-12T00:55:12,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T00:55:12,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372972533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:12,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372972535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:12,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372972646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:12,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:12,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:12,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:12,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372972655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:12,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:12,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:12,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:12,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/8bc6316e3a8542c9b8aeceaef6589bfd 2024-11-12T00:55:12,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/d3fa130180e54401b71c73e48740b6f7 is 50, key is test_row_0/B:col10/1731372912361/Put/seqid=0 2024-11-12T00:55:12,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742324_1500 (size=12301) 2024-11-12T00:55:12,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T00:55:12,856 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:12,856 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:55:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-12T00:55:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T00:55:12,857 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:55:12,858 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:55:12,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:55:12,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372972865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:12,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372972876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:12,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:12,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:12,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T00:55:13,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T00:55:13,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:13,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:13,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:13,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:13,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:13,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T00:55:13,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:13,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:13,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:13,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:13,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:13,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:13,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T00:55:13,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/d3fa130180e54401b71c73e48740b6f7 2024-11-12T00:55:13,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:13,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372973185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:13,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/2aaecbff46ec4fbea9aba124183374df is 50, key is test_row_0/C:col10/1731372912361/Put/seqid=0 2024-11-12T00:55:13,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742325_1501 (size=12301) 2024-11-12T00:55:13,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:13,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372973187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:13,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:13,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:13,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:13,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:13,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372973216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:13,226 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_1' in 2400f2d4915c3c25d637e8ed29b4fadd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:13,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:13,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:13,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:13,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T00:55:13,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:13,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:13,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:13,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:13,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:13,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T00:55:13,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:13,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:13,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:13,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:13,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:13,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T00:55:13,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/2aaecbff46ec4fbea9aba124183374df 2024-11-12T00:55:13,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/8bc6316e3a8542c9b8aeceaef6589bfd as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8bc6316e3a8542c9b8aeceaef6589bfd 2024-11-12T00:55:13,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8bc6316e3a8542c9b8aeceaef6589bfd, entries=150, sequenceid=259, filesize=12.0 K 2024-11-12T00:55:13,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/d3fa130180e54401b71c73e48740b6f7 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/d3fa130180e54401b71c73e48740b6f7 2024-11-12T00:55:13,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/d3fa130180e54401b71c73e48740b6f7, entries=150, sequenceid=259, filesize=12.0 K 2024-11-12T00:55:13,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/2aaecbff46ec4fbea9aba124183374df as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/2aaecbff46ec4fbea9aba124183374df 2024-11-12T00:55:13,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/2aaecbff46ec4fbea9aba124183374df, entries=150, sequenceid=259, filesize=12.0 K 2024-11-12T00:55:13,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 2400f2d4915c3c25d637e8ed29b4fadd in 1255ms, sequenceid=259, compaction requested=true 2024-11-12T00:55:13,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:55:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:13,618 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:55:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:13,618 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:55:13,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:13,619 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:13,619 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:13,619 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/B is initiating minor compaction (all files) 2024-11-12T00:55:13,619 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/A is initiating minor compaction (all files) 2024-11-12T00:55:13,619 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/A in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:13,619 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/B in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:13,619 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/d83f1603ce154d0c8215f37569fd1f9d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/8f1ba4715b7c40a59fe24e7b5340b19b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/d3fa130180e54401b71c73e48740b6f7] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=36.3 K 2024-11-12T00:55:13,619 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/bd6ddd62d48b4a8c87949c3c20048ab9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/d5a9622894374a80824708c534169a87, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8bc6316e3a8542c9b8aeceaef6589bfd] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=36.3 K 2024-11-12T00:55:13,619 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd6ddd62d48b4a8c87949c3c20048ab9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731372908975 2024-11-12T00:55:13,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T00:55:13,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:13,620 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-12T00:55:13,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:13,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:13,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:13,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:13,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:13,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:13,621 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting d83f1603ce154d0c8215f37569fd1f9d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731372908975 2024-11-12T00:55:13,623 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5a9622894374a80824708c534169a87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731372910148 2024-11-12T00:55:13,623 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f1ba4715b7c40a59fe24e7b5340b19b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731372910148 2024-11-12T00:55:13,623 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bc6316e3a8542c9b8aeceaef6589bfd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731372912354 2024-11-12T00:55:13,623 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting d3fa130180e54401b71c73e48740b6f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731372912354 2024-11-12T00:55:13,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/02ecbebeeafa4a19a7f9f6804360fad7 is 50, key is test_row_0/A:col10/1731372912531/Put/seqid=0 2024-11-12T00:55:13,637 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#B#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:13,637 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#A#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:13,637 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/53eebf6776274a9281bd47c2aa3c8e38 is 50, key is test_row_0/B:col10/1731372912361/Put/seqid=0 2024-11-12T00:55:13,640 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/8fa90fca64fd4001a34c4eadb0abf327 is 50, key is test_row_0/A:col10/1731372912361/Put/seqid=0 2024-11-12T00:55:13,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742326_1502 (size=12301) 2024-11-12T00:55:13,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742327_1503 (size=12983) 2024-11-12T00:55:13,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742328_1504 (size=12983) 2024-11-12T00:55:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:13,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:13,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372973741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:13,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:13,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372973742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:13,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:13,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:13,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:13,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372973855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:13,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372973855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T00:55:14,044 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/02ecbebeeafa4a19a7f9f6804360fad7 2024-11-12T00:55:14,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/67c8132ed455489aa8cc51afecb1ef6f is 50, key is test_row_0/B:col10/1731372912531/Put/seqid=0 2024-11-12T00:55:14,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742329_1505 (size=12301) 2024-11-12T00:55:14,054 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T00:55:14,054 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/53eebf6776274a9281bd47c2aa3c8e38 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/53eebf6776274a9281bd47c2aa3c8e38 2024-11-12T00:55:14,058 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/B of 2400f2d4915c3c25d637e8ed29b4fadd into 53eebf6776274a9281bd47c2aa3c8e38(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:14,058 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:14,058 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/B, priority=13, startTime=1731372913618; duration=0sec 2024-11-12T00:55:14,058 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:14,058 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:B 2024-11-12T00:55:14,058 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:14,059 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:14,059 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/8fa90fca64fd4001a34c4eadb0abf327 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8fa90fca64fd4001a34c4eadb0abf327 2024-11-12T00:55:14,059 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/C is initiating minor compaction (all files) 2024-11-12T00:55:14,059 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/C in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:14,059 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/73ef3893f4454e81bbe002ec272c0f26, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/4b6c14ec32fc464ebe850a42fd74ff0c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/2aaecbff46ec4fbea9aba124183374df] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=36.3 K 2024-11-12T00:55:14,059 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 73ef3893f4454e81bbe002ec272c0f26, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731372908975 2024-11-12T00:55:14,060 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b6c14ec32fc464ebe850a42fd74ff0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731372910148 2024-11-12T00:55:14,060 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aaecbff46ec4fbea9aba124183374df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731372912354 2024-11-12T00:55:14,063 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/A of 2400f2d4915c3c25d637e8ed29b4fadd into 8fa90fca64fd4001a34c4eadb0abf327(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:14,063 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:14,063 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/A, priority=13, startTime=1731372913618; duration=0sec 2024-11-12T00:55:14,063 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:14,063 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:A 2024-11-12T00:55:14,066 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#C#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:14,067 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/90afa8f6a1c64afb904e3f3cdd790279 is 50, key is test_row_0/C:col10/1731372912361/Put/seqid=0 2024-11-12T00:55:14,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742330_1506 (size=12983) 2024-11-12T00:55:14,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372974075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:14,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372974075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:14,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372974386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:14,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372974386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:14,454 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/67c8132ed455489aa8cc51afecb1ef6f 2024-11-12T00:55:14,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/89198a6270c1455f8527b6926ab617e4 is 50, key is test_row_0/C:col10/1731372912531/Put/seqid=0 2024-11-12T00:55:14,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742331_1507 (size=12301) 2024-11-12T00:55:14,472 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/90afa8f6a1c64afb904e3f3cdd790279 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/90afa8f6a1c64afb904e3f3cdd790279 2024-11-12T00:55:14,476 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/C of 2400f2d4915c3c25d637e8ed29b4fadd into 90afa8f6a1c64afb904e3f3cdd790279(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:14,476 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:14,476 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/C, priority=13, startTime=1731372913618; duration=0sec 2024-11-12T00:55:14,476 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:14,476 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:C 2024-11-12T00:55:14,863 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/89198a6270c1455f8527b6926ab617e4 2024-11-12T00:55:14,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/02ecbebeeafa4a19a7f9f6804360fad7 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/02ecbebeeafa4a19a7f9f6804360fad7 2024-11-12T00:55:14,868 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/02ecbebeeafa4a19a7f9f6804360fad7, entries=150, sequenceid=296, filesize=12.0 K 2024-11-12T00:55:14,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/67c8132ed455489aa8cc51afecb1ef6f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/67c8132ed455489aa8cc51afecb1ef6f 2024-11-12T00:55:14,871 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/67c8132ed455489aa8cc51afecb1ef6f, entries=150, sequenceid=296, filesize=12.0 K 2024-11-12T00:55:14,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/89198a6270c1455f8527b6926ab617e4 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/89198a6270c1455f8527b6926ab617e4 2024-11-12T00:55:14,875 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/89198a6270c1455f8527b6926ab617e4, entries=150, sequenceid=296, filesize=12.0 K 2024-11-12T00:55:14,876 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 2400f2d4915c3c25d637e8ed29b4fadd in 1255ms, sequenceid=296, compaction requested=false 2024-11-12T00:55:14,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:14,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:14,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-12T00:55:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-12T00:55:14,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-12T00:55:14,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0190 sec 2024-11-12T00:55:14,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.0230 sec 2024-11-12T00:55:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:14,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T00:55:14,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:14,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:14,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:14,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:14,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:14,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:14,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/461088c371674103ae3e3e1afab411c4 is 50, key is test_row_0/A:col10/1731372914898/Put/seqid=0 2024-11-12T00:55:14,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742332_1508 (size=14741) 2024-11-12T00:55:14,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372974969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:14,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731372914943, max timestamp=1731372914972Num uniques 859; 2024-11-12T00:55:14,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731372914943, max timestamp=1731372914972Num uniques 859; 2024-11-12T00:55:14,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:55:14,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:55:14,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731372914943, max timestamp=1731372914972Num uniques 859; 2024-11-12T00:55:14,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:55:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T00:55:15,005 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:15,005 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:55:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-12T00:55:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T00:55:15,007 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:55:15,008 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:55:15,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:55:15,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372975083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372975085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T00:55:15,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T00:55:15,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:15,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:15,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:15,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372975206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:15,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/461088c371674103ae3e3e1afab411c4 2024-11-12T00:55:15,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/2b2a796b9a46457790a05f6f7f6ad26a is 50, key is test_row_0/B:col10/1731372914898/Put/seqid=0 2024-11-12T00:55:15,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T00:55:15,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:15,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:15,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:15,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742333_1509 (size=12301) 2024-11-12T00:55:15,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372975306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:15,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/2b2a796b9a46457790a05f6f7f6ad26a 2024-11-12T00:55:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T00:55:15,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/0bb0e61d02e24c71a07aff825b688e79 is 50, key is test_row_0/C:col10/1731372914898/Put/seqid=0 2024-11-12T00:55:15,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742334_1510 (size=12301) 2024-11-12T00:55:15,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372975426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:15,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T00:55:15,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:15,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:15,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:15,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T00:55:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:15,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372975625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T00:55:15,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/0bb0e61d02e24c71a07aff825b688e79 2024-11-12T00:55:15,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/461088c371674103ae3e3e1afab411c4 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/461088c371674103ae3e3e1afab411c4 2024-11-12T00:55:15,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/461088c371674103ae3e3e1afab411c4, entries=200, sequenceid=310, filesize=14.4 K 2024-11-12T00:55:15,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/2b2a796b9a46457790a05f6f7f6ad26a as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/2b2a796b9a46457790a05f6f7f6ad26a 2024-11-12T00:55:15,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/2b2a796b9a46457790a05f6f7f6ad26a, entries=150, sequenceid=310, filesize=12.0 K 2024-11-12T00:55:15,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/0bb0e61d02e24c71a07aff825b688e79 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/0bb0e61d02e24c71a07aff825b688e79 2024-11-12T00:55:15,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/0bb0e61d02e24c71a07aff825b688e79, entries=150, sequenceid=310, filesize=12.0 K 2024-11-12T00:55:15,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2400f2d4915c3c25d637e8ed29b4fadd in 843ms, sequenceid=310, compaction requested=true 2024-11-12T00:55:15,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:15,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:55:15,742 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:15,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:15,742 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:15,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:55:15,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:15,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:55:15,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:15,743 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:15,743 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:15,743 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/B is initiating minor compaction (all files) 2024-11-12T00:55:15,743 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/A is initiating minor compaction (all files) 2024-11-12T00:55:15,743 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/B in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:15,743 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/A in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:15,743 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/53eebf6776274a9281bd47c2aa3c8e38, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/67c8132ed455489aa8cc51afecb1ef6f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/2b2a796b9a46457790a05f6f7f6ad26a] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=36.7 K 2024-11-12T00:55:15,743 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8fa90fca64fd4001a34c4eadb0abf327, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/02ecbebeeafa4a19a7f9f6804360fad7, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/461088c371674103ae3e3e1afab411c4] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=39.1 K 2024-11-12T00:55:15,744 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 53eebf6776274a9281bd47c2aa3c8e38, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731372912354 2024-11-12T00:55:15,744 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fa90fca64fd4001a34c4eadb0abf327, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731372912354 2024-11-12T00:55:15,744 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02ecbebeeafa4a19a7f9f6804360fad7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731372912510 2024-11-12T00:55:15,744 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 67c8132ed455489aa8cc51afecb1ef6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731372912510 2024-11-12T00:55:15,744 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 461088c371674103ae3e3e1afab411c4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731372913725 2024-11-12T00:55:15,744 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b2a796b9a46457790a05f6f7f6ad26a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731372913725 2024-11-12T00:55:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:15,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-12T00:55:15,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:15,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:15,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:15,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:15,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:15,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:15,751 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#A#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:15,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/21b5fd6c911c43a09d6d659f2a5c36ce is 50, key is test_row_0/A:col10/1731372915745/Put/seqid=0 2024-11-12T00:55:15,751 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/8e9e5982ab9c4bbd8c8a1bd790a74071 is 50, key is test_row_0/A:col10/1731372914898/Put/seqid=0 2024-11-12T00:55:15,754 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#B#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:15,754 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/dfe878a585804703936ecf1cd2d10360 is 50, key is test_row_0/B:col10/1731372914898/Put/seqid=0 2024-11-12T00:55:15,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742335_1511 (size=13085) 2024-11-12T00:55:15,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742336_1512 (size=14741) 2024-11-12T00:55:15,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742337_1513 (size=13085) 2024-11-12T00:55:15,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T00:55:15,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:15,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:15,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:15,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,780 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/dfe878a585804703936ecf1cd2d10360 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/dfe878a585804703936ecf1cd2d10360 2024-11-12T00:55:15,784 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/B of 2400f2d4915c3c25d637e8ed29b4fadd into dfe878a585804703936ecf1cd2d10360(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:15,784 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:15,784 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/B, priority=13, startTime=1731372915742; duration=0sec 2024-11-12T00:55:15,784 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:15,784 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:B 2024-11-12T00:55:15,784 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:15,785 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:15,785 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/C is initiating minor compaction (all files) 2024-11-12T00:55:15,785 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/C in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:15,785 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/90afa8f6a1c64afb904e3f3cdd790279, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/89198a6270c1455f8527b6926ab617e4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/0bb0e61d02e24c71a07aff825b688e79] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=36.7 K 2024-11-12T00:55:15,786 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 90afa8f6a1c64afb904e3f3cdd790279, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731372912354 2024-11-12T00:55:15,786 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 89198a6270c1455f8527b6926ab617e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731372912510 2024-11-12T00:55:15,786 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bb0e61d02e24c71a07aff825b688e79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731372913725 2024-11-12T00:55:15,790 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#C#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:15,791 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/9f7bfb870d694d1bbd8e017f5bd49e53 is 50, key is test_row_0/C:col10/1731372914898/Put/seqid=0 2024-11-12T00:55:15,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742338_1514 (size=13085) 2024-11-12T00:55:15,807 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/9f7bfb870d694d1bbd8e017f5bd49e53 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/9f7bfb870d694d1bbd8e017f5bd49e53 2024-11-12T00:55:15,811 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/C of 2400f2d4915c3c25d637e8ed29b4fadd into 9f7bfb870d694d1bbd8e017f5bd49e53(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:15,811 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:15,811 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/C, priority=13, startTime=1731372915742; duration=0sec 2024-11-12T00:55:15,811 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:15,811 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:C 2024-11-12T00:55:15,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372975814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:15,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T00:55:15,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:15,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:15,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:15,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:15,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372975936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:16,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T00:55:16,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:16,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:16,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:16,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T00:55:16,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372976145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:16,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372976146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:16,164 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/8e9e5982ab9c4bbd8c8a1bd790a74071 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8e9e5982ab9c4bbd8c8a1bd790a74071 2024-11-12T00:55:16,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/21b5fd6c911c43a09d6d659f2a5c36ce 2024-11-12T00:55:16,167 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/A of 2400f2d4915c3c25d637e8ed29b4fadd into 8e9e5982ab9c4bbd8c8a1bd790a74071(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:16,167 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:16,167 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/A, priority=13, startTime=1731372915742; duration=0sec 2024-11-12T00:55:16,167 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:16,168 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:A 2024-11-12T00:55:16,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/e325368220fb4aa0abcb101b1c9dd09a is 50, key is test_row_0/B:col10/1731372915745/Put/seqid=0 2024-11-12T00:55:16,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742339_1515 (size=12301) 2024-11-12T00:55:16,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T00:55:16,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:16,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:16,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:16,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T00:55:16,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:16,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:16,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:16,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372976465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:16,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T00:55:16,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:16,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:16,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:16,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/e325368220fb4aa0abcb101b1c9dd09a 2024-11-12T00:55:16,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/7a29ba6453f54d4c876756a50fbb6375 is 50, key is test_row_0/C:col10/1731372915745/Put/seqid=0 2024-11-12T00:55:16,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742340_1516 (size=12301) 2024-11-12T00:55:16,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T00:55:16,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:16,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:16,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:16,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T00:55:16,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:16,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:16,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:16,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:16,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39678 deadline: 1731372976946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:16,951 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 2400f2d4915c3c25d637e8ed29b4fadd of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:16,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372976985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:16,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/7a29ba6453f54d4c876756a50fbb6375 2024-11-12T00:55:16,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/21b5fd6c911c43a09d6d659f2a5c36ce as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/21b5fd6c911c43a09d6d659f2a5c36ce 2024-11-12T00:55:16,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/21b5fd6c911c43a09d6d659f2a5c36ce, entries=200, sequenceid=346, filesize=14.4 K 2024-11-12T00:55:16,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/e325368220fb4aa0abcb101b1c9dd09a as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e325368220fb4aa0abcb101b1c9dd09a 2024-11-12T00:55:17,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T00:55:17,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:17,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:17,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:17,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:17,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:17,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e325368220fb4aa0abcb101b1c9dd09a, entries=150, sequenceid=346, filesize=12.0 K 2024-11-12T00:55:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:17,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/7a29ba6453f54d4c876756a50fbb6375 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/7a29ba6453f54d4c876756a50fbb6375 2024-11-12T00:55:17,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/7a29ba6453f54d4c876756a50fbb6375, entries=150, sequenceid=346, filesize=12.0 K 2024-11-12T00:55:17,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 2400f2d4915c3c25d637e8ed29b4fadd in 1260ms, sequenceid=346, compaction requested=false 2024-11-12T00:55:17,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:17,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T00:55:17,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:17,154 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T00:55:17,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:17,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:17,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:17,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:17,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:17,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T00:55:17,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/3af80078452840b68993c6af0c235f31 is 50, key is test_row_0/A:col10/1731372915804/Put/seqid=0 2024-11-12T00:55:17,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742341_1517 (size=12301) 2024-11-12T00:55:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:17,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. as already flushing 2024-11-12T00:55:17,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731372917197, max timestamp=1731372917256Num uniques 859; 2024-11-12T00:55:17,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T00:55:17,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731372917197, max timestamp=1731372917256Num uniques 859; 2024-11-12T00:55:17,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731372917197, max timestamp=1731372917256Num uniques 859; 2024-11-12T00:55:17,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T00:55:17,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T00:55:17,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372977317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:17,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:17,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372977324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:17,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:17,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:17,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372977436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:17,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372977436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:17,564 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/3af80078452840b68993c6af0c235f31 2024-11-12T00:55:17,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/bd0f9f61202249a3a614f67dd23752d5 is 50, key is test_row_0/B:col10/1731372915804/Put/seqid=0 2024-11-12T00:55:17,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742342_1518 (size=12301) 2024-11-12T00:55:17,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372977655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:17,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372977656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:17,972 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/bd0f9f61202249a3a614f67dd23752d5 2024-11-12T00:55:17,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/222224ae470042baa3603d089b84534c is 50, key is test_row_0/C:col10/1731372915804/Put/seqid=0 2024-11-12T00:55:17,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372977975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:17,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372977975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:17,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742343_1519 (size=12301) 2024-11-12T00:55:17,989 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/222224ae470042baa3603d089b84534c 2024-11-12T00:55:17,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/3af80078452840b68993c6af0c235f31 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/3af80078452840b68993c6af0c235f31 2024-11-12T00:55:17,995 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/3af80078452840b68993c6af0c235f31, entries=150, sequenceid=359, filesize=12.0 K 2024-11-12T00:55:17,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/bd0f9f61202249a3a614f67dd23752d5 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/bd0f9f61202249a3a614f67dd23752d5 2024-11-12T00:55:17,998 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/bd0f9f61202249a3a614f67dd23752d5, entries=150, sequenceid=359, filesize=12.0 K 2024-11-12T00:55:17,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/222224ae470042baa3603d089b84534c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/222224ae470042baa3603d089b84534c 2024-11-12T00:55:18,001 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/222224ae470042baa3603d089b84534c, entries=150, sequenceid=359, filesize=12.0 K 2024-11-12T00:55:18,001 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 2400f2d4915c3c25d637e8ed29b4fadd in 847ms, sequenceid=359, compaction requested=true 2024-11-12T00:55:18,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:18,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:18,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-12T00:55:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-12T00:55:18,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-12T00:55:18,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9940 sec 2024-11-12T00:55:18,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.9980 sec 2024-11-12T00:55:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:18,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-12T00:55:18,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:18,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:18,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:18,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:18,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:18,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:18,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/343b3475f26b4b459212eeec93dcf916 is 50, key is test_row_0/A:col10/1731372918006/Put/seqid=0 2024-11-12T00:55:18,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742344_1520 (size=14741) 2024-11-12T00:55:18,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372978053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:18,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372978165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:18,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372978375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:18,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/343b3475f26b4b459212eeec93dcf916 2024-11-12T00:55:18,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/a9247176fb66490480e393c582e7eac8 is 50, key is test_row_0/B:col10/1731372918006/Put/seqid=0 2024-11-12T00:55:18,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742345_1521 (size=12301) 2024-11-12T00:55:18,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39608 deadline: 1731372978495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:18,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39638 deadline: 1731372978495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:18,635 INFO [Thread-2000 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2000. 2024-11-12T00:55:18,635 DEBUG [Thread-2000 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:18,635 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:18,635 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:18,635 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:18,636 INFO [Thread-1998 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1998. 2024-11-12T00:55:18,636 DEBUG [Thread-1998 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:18,637 DEBUG [Thread-1998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:18,637 DEBUG [Thread-1998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:18,637 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:18,638 INFO [Thread-2004 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2004. 2024-11-12T00:55:18,638 DEBUG [Thread-2004 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:18,638 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:18,638 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:18,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:18,647 INFO [Thread-2002 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2002. 2024-11-12T00:55:18,647 DEBUG [Thread-2002 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:18,647 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:18,647 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:18,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:18,648 INFO [Thread-1996 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1996. 2024-11-12T00:55:18,648 DEBUG [Thread-1996 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:18,648 DEBUG [Thread-1996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:18,648 DEBUG [Thread-1996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:18,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:18,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372978685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:18,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/a9247176fb66490480e393c582e7eac8 2024-11-12T00:55:18,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/cadc6e856d5b4bcba54005bad7dcc711 is 50, key is test_row_0/C:col10/1731372918006/Put/seqid=0 2024-11-12T00:55:18,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742346_1522 (size=12301) 2024-11-12T00:55:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T00:55:19,177 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:19,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39654 deadline: 1731372979196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 , the old value is region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., hostname=6928b69dca1b,33953,1731372765559, seqNum=2 because the exception is null or not the one we care about 2024-11-12T00:55:19,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/cadc6e856d5b4bcba54005bad7dcc711 2024-11-12T00:55:19,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/343b3475f26b4b459212eeec93dcf916 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/343b3475f26b4b459212eeec93dcf916 2024-11-12T00:55:19,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/343b3475f26b4b459212eeec93dcf916, entries=200, sequenceid=395, filesize=14.4 K 2024-11-12T00:55:19,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/a9247176fb66490480e393c582e7eac8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a9247176fb66490480e393c582e7eac8 2024-11-12T00:55:19,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a9247176fb66490480e393c582e7eac8, entries=150, sequenceid=395, filesize=12.0 K 2024-11-12T00:55:19,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/cadc6e856d5b4bcba54005bad7dcc711 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/cadc6e856d5b4bcba54005bad7dcc711 2024-11-12T00:55:19,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/cadc6e856d5b4bcba54005bad7dcc711, entries=150, sequenceid=395, filesize=12.0 K 2024-11-12T00:55:19,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 2400f2d4915c3c25d637e8ed29b4fadd in 1261ms, sequenceid=395, compaction requested=true 2024-11-12T00:55:19,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:19,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:55:19,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:19,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:55:19,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:19,267 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:55:19,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2400f2d4915c3c25d637e8ed29b4fadd:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:55:19,267 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:55:19,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:19,268 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:55:19,268 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54868 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:55:19,268 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/B is initiating minor compaction (all files) 2024-11-12T00:55:19,269 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/A is initiating minor compaction (all files) 2024-11-12T00:55:19,269 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/B in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:19,269 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/A in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:19,269 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/dfe878a585804703936ecf1cd2d10360, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e325368220fb4aa0abcb101b1c9dd09a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/bd0f9f61202249a3a614f67dd23752d5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a9247176fb66490480e393c582e7eac8] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=48.8 K 2024-11-12T00:55:19,269 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8e9e5982ab9c4bbd8c8a1bd790a74071, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/21b5fd6c911c43a09d6d659f2a5c36ce, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/3af80078452840b68993c6af0c235f31, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/343b3475f26b4b459212eeec93dcf916] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=53.6 K 2024-11-12T00:55:19,269 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe878a585804703936ecf1cd2d10360, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731372913725 2024-11-12T00:55:19,269 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e9e5982ab9c4bbd8c8a1bd790a74071, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731372913725 2024-11-12T00:55:19,269 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting e325368220fb4aa0abcb101b1c9dd09a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731372915060 2024-11-12T00:55:19,269 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21b5fd6c911c43a09d6d659f2a5c36ce, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731372915060 2024-11-12T00:55:19,270 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting bd0f9f61202249a3a614f67dd23752d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731372915796 2024-11-12T00:55:19,270 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3af80078452840b68993c6af0c235f31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731372915796 2024-11-12T00:55:19,270 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting a9247176fb66490480e393c582e7eac8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731372917303 2024-11-12T00:55:19,270 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 343b3475f26b4b459212eeec93dcf916, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731372917303 2024-11-12T00:55:19,277 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#B#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:19,277 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#A#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:19,278 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/d9d6e14ffd074cdeb6414f92da028c2a is 50, key is test_row_0/A:col10/1731372918006/Put/seqid=0 2024-11-12T00:55:19,278 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/04bc17a4ee4741a4b5446739c3543643 is 50, key is test_row_0/B:col10/1731372918006/Put/seqid=0 2024-11-12T00:55:19,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742347_1523 (size=13221) 2024-11-12T00:55:19,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742348_1524 (size=13221) 2024-11-12T00:55:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:19,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T00:55:19,519 INFO [Thread-1987 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1987. 2024-11-12T00:55:19,520 INFO [Thread-1989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1989. 2024-11-12T00:55:19,520 DEBUG [Thread-1987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:19,520 DEBUG [Thread-1989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:19,520 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:19,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:19,520 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:19,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:19,520 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:19,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:19,520 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:19,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:19,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:19,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:19,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:19,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:19,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/de0ed0829c6e4386bc9b2ef635cc6c3c is 50, key is test_row_0/A:col10/1731372918051/Put/seqid=0 2024-11-12T00:55:19,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742349_1525 (size=12301) 2024-11-12T00:55:19,684 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/d9d6e14ffd074cdeb6414f92da028c2a as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/d9d6e14ffd074cdeb6414f92da028c2a 2024-11-12T00:55:19,684 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/04bc17a4ee4741a4b5446739c3543643 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/04bc17a4ee4741a4b5446739c3543643 2024-11-12T00:55:19,687 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/B of 2400f2d4915c3c25d637e8ed29b4fadd into 04bc17a4ee4741a4b5446739c3543643(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:19,687 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/A of 2400f2d4915c3c25d637e8ed29b4fadd into d9d6e14ffd074cdeb6414f92da028c2a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:19,687 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:19,687 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:19,687 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/A, priority=12, startTime=1731372919267; duration=0sec 2024-11-12T00:55:19,687 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/B, priority=12, startTime=1731372919267; duration=0sec 2024-11-12T00:55:19,687 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:19,687 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:19,687 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:A 2024-11-12T00:55:19,687 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:B 2024-11-12T00:55:19,687 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T00:55:19,688 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T00:55:19,688 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 2400f2d4915c3c25d637e8ed29b4fadd/C is initiating minor compaction (all files) 2024-11-12T00:55:19,688 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2400f2d4915c3c25d637e8ed29b4fadd/C in TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:19,688 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/9f7bfb870d694d1bbd8e017f5bd49e53, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/7a29ba6453f54d4c876756a50fbb6375, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/222224ae470042baa3603d089b84534c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/cadc6e856d5b4bcba54005bad7dcc711] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp, totalSize=48.8 K 2024-11-12T00:55:19,689 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f7bfb870d694d1bbd8e017f5bd49e53, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731372913725 2024-11-12T00:55:19,689 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a29ba6453f54d4c876756a50fbb6375, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731372915060 2024-11-12T00:55:19,689 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 222224ae470042baa3603d089b84534c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731372915796 2024-11-12T00:55:19,689 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting cadc6e856d5b4bcba54005bad7dcc711, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731372917303 2024-11-12T00:55:19,694 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2400f2d4915c3c25d637e8ed29b4fadd#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:19,695 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/145c62a9586d466592167268b756f45e is 50, key is test_row_0/C:col10/1731372918006/Put/seqid=0 2024-11-12T00:55:19,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742350_1526 (size=13221) 2024-11-12T00:55:19,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/de0ed0829c6e4386bc9b2ef635cc6c3c 2024-11-12T00:55:19,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/294d3082e8fc49af81fbc2ad3cf03f00 is 50, key is test_row_0/B:col10/1731372918051/Put/seqid=0 2024-11-12T00:55:19,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742351_1527 (size=12301) 2024-11-12T00:55:20,109 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/145c62a9586d466592167268b756f45e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/145c62a9586d466592167268b756f45e 2024-11-12T00:55:20,114 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2400f2d4915c3c25d637e8ed29b4fadd/C of 2400f2d4915c3c25d637e8ed29b4fadd into 145c62a9586d466592167268b756f45e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:20,114 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:20,114 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., storeName=2400f2d4915c3c25d637e8ed29b4fadd/C, priority=12, startTime=1731372919267; duration=0sec 2024-11-12T00:55:20,114 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:20,114 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2400f2d4915c3c25d637e8ed29b4fadd:C 2024-11-12T00:55:20,210 INFO [Thread-1993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1993. 2024-11-12T00:55:20,210 DEBUG [Thread-1993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:20,210 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:20,211 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:20,211 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:20,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/294d3082e8fc49af81fbc2ad3cf03f00 2024-11-12T00:55:20,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/2403f2262666476dbe5adad2c8574f2d is 50, key is test_row_0/C:col10/1731372918051/Put/seqid=0 2024-11-12T00:55:20,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742352_1528 (size=12301) 2024-11-12T00:55:20,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/2403f2262666476dbe5adad2c8574f2d 2024-11-12T00:55:20,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/de0ed0829c6e4386bc9b2ef635cc6c3c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/de0ed0829c6e4386bc9b2ef635cc6c3c 2024-11-12T00:55:20,767 INFO [Thread-1985 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1985. 2024-11-12T00:55:20,768 DEBUG [Thread-1985 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:20,768 DEBUG [Thread-1985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:20,768 DEBUG [Thread-1985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:20,768 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:20,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/de0ed0829c6e4386bc9b2ef635cc6c3c, entries=150, sequenceid=406, filesize=12.0 K 2024-11-12T00:55:20,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/294d3082e8fc49af81fbc2ad3cf03f00 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/294d3082e8fc49af81fbc2ad3cf03f00 2024-11-12T00:55:20,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/294d3082e8fc49af81fbc2ad3cf03f00, entries=150, sequenceid=406, filesize=12.0 K 2024-11-12T00:55:20,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/2403f2262666476dbe5adad2c8574f2d as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/2403f2262666476dbe5adad2c8574f2d 2024-11-12T00:55:20,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/2403f2262666476dbe5adad2c8574f2d, entries=150, sequenceid=406, filesize=12.0 K 2024-11-12T00:55:20,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=13.42 KB/13740 for 2400f2d4915c3c25d637e8ed29b4fadd in 1261ms, sequenceid=406, compaction requested=false 2024-11-12T00:55:20,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2400f2d4915c3c25d637e8ed29b4fadd: 2024-11-12T00:55:27,049 INFO [Thread-1991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1991. 2024-11-12T00:55:27,049 DEBUG [Thread-1991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:27,049 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:27,049 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:27,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 5 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1338 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4008 rows 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1337 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4007 rows 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1336 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4007 rows 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1339 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4013 rows 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1340 2024-11-12T00:55:27,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4019 rows 2024-11-12T00:55:27,051 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T00:55:27,051 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T00:55:27,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:27,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:27,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:27,053 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T00:55:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T00:55:27,060 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372927059"}]},"ts":"1731372927059"} 2024-11-12T00:55:27,061 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T00:55:27,061 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T00:55:27,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T00:55:27,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2400f2d4915c3c25d637e8ed29b4fadd, UNASSIGN}] 2024-11-12T00:55:27,064 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2400f2d4915c3c25d637e8ed29b4fadd, UNASSIGN 2024-11-12T00:55:27,065 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=2400f2d4915c3c25d637e8ed29b4fadd, regionState=CLOSING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:27,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2400f2d4915c3c25d637e8ed29b4fadd, UNASSIGN because future has completed 2024-11-12T00:55:27,066 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T00:55:27,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:55:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T00:55:27,220 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(122): Close 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:27,220 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T00:55:27,220 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1722): Closing 2400f2d4915c3c25d637e8ed29b4fadd, disabling compactions & flushes 2024-11-12T00:55:27,220 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:27,220 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:27,220 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. after waiting 0 ms 2024-11-12T00:55:27,220 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:27,221 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(2902): Flushing 2400f2d4915c3c25d637e8ed29b4fadd 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-12T00:55:27,221 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=A 2024-11-12T00:55:27,221 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:27,221 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=B 2024-11-12T00:55:27,222 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:27,222 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2400f2d4915c3c25d637e8ed29b4fadd, store=C 2024-11-12T00:55:27,222 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:27,230 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/f6c5ad4338d2412b9fa0fe2c3242eddc is 50, key is test_row_0/A:col10/1731372927046/Put/seqid=0 2024-11-12T00:55:27,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742353_1529 (size=9857) 2024-11-12T00:55:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T00:55:27,634 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/f6c5ad4338d2412b9fa0fe2c3242eddc 2024-11-12T00:55:27,649 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/9f7d1cdd1c854d628b4af477bf667008 is 50, key is test_row_0/B:col10/1731372927046/Put/seqid=0 2024-11-12T00:55:27,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742354_1530 (size=9857) 2024-11-12T00:55:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T00:55:28,056 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/9f7d1cdd1c854d628b4af477bf667008 2024-11-12T00:55:28,068 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/d4b2965e4f974641a18097abcddab1a4 is 50, key is test_row_0/C:col10/1731372927046/Put/seqid=0 2024-11-12T00:55:28,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742355_1531 (size=9857) 2024-11-12T00:55:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T00:55:28,473 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/d4b2965e4f974641a18097abcddab1a4 2024-11-12T00:55:28,482 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/A/f6c5ad4338d2412b9fa0fe2c3242eddc as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/f6c5ad4338d2412b9fa0fe2c3242eddc 2024-11-12T00:55:28,487 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/f6c5ad4338d2412b9fa0fe2c3242eddc, entries=100, sequenceid=415, filesize=9.6 K 2024-11-12T00:55:28,488 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/B/9f7d1cdd1c854d628b4af477bf667008 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/9f7d1cdd1c854d628b4af477bf667008 2024-11-12T00:55:28,491 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/9f7d1cdd1c854d628b4af477bf667008, entries=100, sequenceid=415, filesize=9.6 K 2024-11-12T00:55:28,492 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/.tmp/C/d4b2965e4f974641a18097abcddab1a4 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/d4b2965e4f974641a18097abcddab1a4 2024-11-12T00:55:28,495 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/d4b2965e4f974641a18097abcddab1a4, entries=100, sequenceid=415, filesize=9.6 K 2024-11-12T00:55:28,495 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 2400f2d4915c3c25d637e8ed29b4fadd in 1275ms, sequenceid=415, compaction requested=true 2024-11-12T00:55:28,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/18bbb76ed13a43fbb98309ef36ad9422, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/a42839bd7b724c82ba1fc9eb19487bd1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/0630ff65950e4c4f9bc151825af1bd32, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/076aca0f9f8943b2a9421a6a4535c45e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/790e2af24aef49aca5c8f5992afdd49f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8b93ec99eb4849bb86157deb862aa0d0, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/7e5500c917ac4f44a9915c232827a4f8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/1719eaa661314122a61adce2b1b607d7, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/760ce223f7e540c2ab2233b26f8b60da, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/57ec1a4041984adcbe7503ff59d75f10, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8424ca1d7d044ab5a900b1fcdceef439, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/4f0c9920c00e435fa7c11b009baa4835, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/da0d98de76c243dbb835b157bdd2b7dd, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8487a74c12b74757b1e8a3aff62e4098, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/f0feda69f8734b7abd2f4d4403186407, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/ba6e6b1624e14967a0ecf7ca694c36a9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/6514a1d11dc343fd9351e068545b002f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/bd6ddd62d48b4a8c87949c3c20048ab9, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/d5a9622894374a80824708c534169a87, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8fa90fca64fd4001a34c4eadb0abf327, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8bc6316e3a8542c9b8aeceaef6589bfd, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/02ecbebeeafa4a19a7f9f6804360fad7, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/461088c371674103ae3e3e1afab411c4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8e9e5982ab9c4bbd8c8a1bd790a74071, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/21b5fd6c911c43a09d6d659f2a5c36ce, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/3af80078452840b68993c6af0c235f31, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/343b3475f26b4b459212eeec93dcf916] to archive 2024-11-12T00:55:28,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:55:28,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/18bbb76ed13a43fbb98309ef36ad9422 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/18bbb76ed13a43fbb98309ef36ad9422 2024-11-12T00:55:28,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/a42839bd7b724c82ba1fc9eb19487bd1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/a42839bd7b724c82ba1fc9eb19487bd1 2024-11-12T00:55:28,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/0630ff65950e4c4f9bc151825af1bd32 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/0630ff65950e4c4f9bc151825af1bd32 2024-11-12T00:55:28,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/076aca0f9f8943b2a9421a6a4535c45e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/076aca0f9f8943b2a9421a6a4535c45e 2024-11-12T00:55:28,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/790e2af24aef49aca5c8f5992afdd49f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/790e2af24aef49aca5c8f5992afdd49f 2024-11-12T00:55:28,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8b93ec99eb4849bb86157deb862aa0d0 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8b93ec99eb4849bb86157deb862aa0d0 2024-11-12T00:55:28,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/7e5500c917ac4f44a9915c232827a4f8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/7e5500c917ac4f44a9915c232827a4f8 2024-11-12T00:55:28,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/1719eaa661314122a61adce2b1b607d7 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/1719eaa661314122a61adce2b1b607d7 2024-11-12T00:55:28,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/760ce223f7e540c2ab2233b26f8b60da to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/760ce223f7e540c2ab2233b26f8b60da 2024-11-12T00:55:28,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/57ec1a4041984adcbe7503ff59d75f10 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/57ec1a4041984adcbe7503ff59d75f10 2024-11-12T00:55:28,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8424ca1d7d044ab5a900b1fcdceef439 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8424ca1d7d044ab5a900b1fcdceef439 2024-11-12T00:55:28,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/4f0c9920c00e435fa7c11b009baa4835 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/4f0c9920c00e435fa7c11b009baa4835 2024-11-12T00:55:28,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/da0d98de76c243dbb835b157bdd2b7dd to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/da0d98de76c243dbb835b157bdd2b7dd 2024-11-12T00:55:28,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8487a74c12b74757b1e8a3aff62e4098 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8487a74c12b74757b1e8a3aff62e4098 2024-11-12T00:55:28,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/f0feda69f8734b7abd2f4d4403186407 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/f0feda69f8734b7abd2f4d4403186407 2024-11-12T00:55:28,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/ba6e6b1624e14967a0ecf7ca694c36a9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/ba6e6b1624e14967a0ecf7ca694c36a9 2024-11-12T00:55:28,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/6514a1d11dc343fd9351e068545b002f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/6514a1d11dc343fd9351e068545b002f 2024-11-12T00:55:28,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/bd6ddd62d48b4a8c87949c3c20048ab9 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/bd6ddd62d48b4a8c87949c3c20048ab9 2024-11-12T00:55:28,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/d5a9622894374a80824708c534169a87 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/d5a9622894374a80824708c534169a87 2024-11-12T00:55:28,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8fa90fca64fd4001a34c4eadb0abf327 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8fa90fca64fd4001a34c4eadb0abf327 2024-11-12T00:55:28,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8bc6316e3a8542c9b8aeceaef6589bfd to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8bc6316e3a8542c9b8aeceaef6589bfd 2024-11-12T00:55:28,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/02ecbebeeafa4a19a7f9f6804360fad7 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/02ecbebeeafa4a19a7f9f6804360fad7 2024-11-12T00:55:28,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/461088c371674103ae3e3e1afab411c4 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/461088c371674103ae3e3e1afab411c4 2024-11-12T00:55:28,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8e9e5982ab9c4bbd8c8a1bd790a74071 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/8e9e5982ab9c4bbd8c8a1bd790a74071 2024-11-12T00:55:28,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/21b5fd6c911c43a09d6d659f2a5c36ce to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/21b5fd6c911c43a09d6d659f2a5c36ce 2024-11-12T00:55:28,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/3af80078452840b68993c6af0c235f31 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/3af80078452840b68993c6af0c235f31 2024-11-12T00:55:28,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/343b3475f26b4b459212eeec93dcf916 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/343b3475f26b4b459212eeec93dcf916 2024-11-12T00:55:28,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a09c4b85321246bf85ec9c416cf61474, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/85000c30178e49b3895e7af38e559f82, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/bcd8bd5e3d6c4b20acde20debd86155f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/7766fbf116c94964b023c3a83c5ab848, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/15c4f1a07a5a4830a96d8ee5605b2ffa, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e5013f93a72b40468b093677f07b6f67, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/3e66a0fca0164105809bbef613b9c671, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/f129a8b4545e4c57b7f8883428b1174b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/56c3662b53f246b09d94dea9a4caca11, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e68a531add0d457cb9bc6f98700e9f01, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a740b10ddd9049d6bfe8f5cd975a1bd5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b08b4b8210c94b52bb6d2ab953959413, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b2aa834ce30c40ccbb35ca285db51044, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/0d9368d304e34c3d97fd90c7d5bb69ab, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/7aa701a151ee499283d210571ce7243d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/f1ef93a1f01f485bab459ff06f971c0b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/d83f1603ce154d0c8215f37569fd1f9d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b33a2e8864334d6e8e2af52af4723163, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/8f1ba4715b7c40a59fe24e7b5340b19b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/53eebf6776274a9281bd47c2aa3c8e38, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/d3fa130180e54401b71c73e48740b6f7, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/67c8132ed455489aa8cc51afecb1ef6f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/dfe878a585804703936ecf1cd2d10360, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/2b2a796b9a46457790a05f6f7f6ad26a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e325368220fb4aa0abcb101b1c9dd09a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/bd0f9f61202249a3a614f67dd23752d5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a9247176fb66490480e393c582e7eac8] to archive 2024-11-12T00:55:28,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:55:28,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a09c4b85321246bf85ec9c416cf61474 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a09c4b85321246bf85ec9c416cf61474 2024-11-12T00:55:28,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/85000c30178e49b3895e7af38e559f82 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/85000c30178e49b3895e7af38e559f82 2024-11-12T00:55:28,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/bcd8bd5e3d6c4b20acde20debd86155f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/bcd8bd5e3d6c4b20acde20debd86155f 2024-11-12T00:55:28,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/7766fbf116c94964b023c3a83c5ab848 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/7766fbf116c94964b023c3a83c5ab848 2024-11-12T00:55:28,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/15c4f1a07a5a4830a96d8ee5605b2ffa to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/15c4f1a07a5a4830a96d8ee5605b2ffa 2024-11-12T00:55:28,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e5013f93a72b40468b093677f07b6f67 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e5013f93a72b40468b093677f07b6f67 2024-11-12T00:55:28,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/3e66a0fca0164105809bbef613b9c671 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/3e66a0fca0164105809bbef613b9c671 2024-11-12T00:55:28,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/f129a8b4545e4c57b7f8883428b1174b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/f129a8b4545e4c57b7f8883428b1174b 2024-11-12T00:55:28,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/56c3662b53f246b09d94dea9a4caca11 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/56c3662b53f246b09d94dea9a4caca11 2024-11-12T00:55:28,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e68a531add0d457cb9bc6f98700e9f01 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e68a531add0d457cb9bc6f98700e9f01 2024-11-12T00:55:28,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a740b10ddd9049d6bfe8f5cd975a1bd5 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a740b10ddd9049d6bfe8f5cd975a1bd5 2024-11-12T00:55:28,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b08b4b8210c94b52bb6d2ab953959413 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b08b4b8210c94b52bb6d2ab953959413 2024-11-12T00:55:28,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b2aa834ce30c40ccbb35ca285db51044 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b2aa834ce30c40ccbb35ca285db51044 2024-11-12T00:55:28,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/0d9368d304e34c3d97fd90c7d5bb69ab to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/0d9368d304e34c3d97fd90c7d5bb69ab 2024-11-12T00:55:28,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/7aa701a151ee499283d210571ce7243d to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/7aa701a151ee499283d210571ce7243d 2024-11-12T00:55:28,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/f1ef93a1f01f485bab459ff06f971c0b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/f1ef93a1f01f485bab459ff06f971c0b 2024-11-12T00:55:28,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/d83f1603ce154d0c8215f37569fd1f9d to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/d83f1603ce154d0c8215f37569fd1f9d 2024-11-12T00:55:28,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b33a2e8864334d6e8e2af52af4723163 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/b33a2e8864334d6e8e2af52af4723163 2024-11-12T00:55:28,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/8f1ba4715b7c40a59fe24e7b5340b19b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/8f1ba4715b7c40a59fe24e7b5340b19b 2024-11-12T00:55:28,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/53eebf6776274a9281bd47c2aa3c8e38 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/53eebf6776274a9281bd47c2aa3c8e38 2024-11-12T00:55:28,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/d3fa130180e54401b71c73e48740b6f7 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/d3fa130180e54401b71c73e48740b6f7 2024-11-12T00:55:28,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/67c8132ed455489aa8cc51afecb1ef6f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/67c8132ed455489aa8cc51afecb1ef6f 2024-11-12T00:55:28,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/dfe878a585804703936ecf1cd2d10360 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/dfe878a585804703936ecf1cd2d10360 2024-11-12T00:55:28,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/2b2a796b9a46457790a05f6f7f6ad26a to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/2b2a796b9a46457790a05f6f7f6ad26a 2024-11-12T00:55:28,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e325368220fb4aa0abcb101b1c9dd09a to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/e325368220fb4aa0abcb101b1c9dd09a 2024-11-12T00:55:28,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/bd0f9f61202249a3a614f67dd23752d5 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/bd0f9f61202249a3a614f67dd23752d5 2024-11-12T00:55:28,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a9247176fb66490480e393c582e7eac8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/a9247176fb66490480e393c582e7eac8 2024-11-12T00:55:28,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/1ce125510b234f38afd958b71bb2723e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/6a8d31e140554b39a6b1556fb682ee87, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/5be8562ac1f14a7db4fcaa069df1212e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/519615f5e9a443abaded6a44fbb8ac1e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/eb0fe16cfe8f4cc2a1e4c753fef0ce24, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/84b7cde577d048dfaee5509291c5b1db, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/4aa6f956b81d4988816da11fbcebd0d6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/f279f6f236d24fb9b220807f93beafc4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/6cacdc7c7d014b19998c6ba06f795411, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/91481ab0f2de4059a732d84dc50009d2, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/bbe2436b2aa44b979bebe66de633f228, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/7e9f1f40084a4ac2a2f4fbfaffb1e3ce, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/aeeb4175d70d4cbeb56162be2df1624d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/8b65f7a267574d55b2469761229a9c7b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/c3c06a1cb8c64632abcaa11933793ce2, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/0d615d75296f4b88b9ea2d8bfc6faea6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/73ef3893f4454e81bbe002ec272c0f26, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/d82754ba30f346d393f4cd67477c3e43, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/4b6c14ec32fc464ebe850a42fd74ff0c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/90afa8f6a1c64afb904e3f3cdd790279, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/2aaecbff46ec4fbea9aba124183374df, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/89198a6270c1455f8527b6926ab617e4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/9f7bfb870d694d1bbd8e017f5bd49e53, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/0bb0e61d02e24c71a07aff825b688e79, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/7a29ba6453f54d4c876756a50fbb6375, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/222224ae470042baa3603d089b84534c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/cadc6e856d5b4bcba54005bad7dcc711] to archive 2024-11-12T00:55:28,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:55:28,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/1ce125510b234f38afd958b71bb2723e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/1ce125510b234f38afd958b71bb2723e 2024-11-12T00:55:28,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/6a8d31e140554b39a6b1556fb682ee87 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/6a8d31e140554b39a6b1556fb682ee87 2024-11-12T00:55:28,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/5be8562ac1f14a7db4fcaa069df1212e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/5be8562ac1f14a7db4fcaa069df1212e 2024-11-12T00:55:28,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/519615f5e9a443abaded6a44fbb8ac1e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/519615f5e9a443abaded6a44fbb8ac1e 2024-11-12T00:55:28,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/eb0fe16cfe8f4cc2a1e4c753fef0ce24 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/eb0fe16cfe8f4cc2a1e4c753fef0ce24 2024-11-12T00:55:28,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/84b7cde577d048dfaee5509291c5b1db to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/84b7cde577d048dfaee5509291c5b1db 2024-11-12T00:55:28,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/4aa6f956b81d4988816da11fbcebd0d6 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/4aa6f956b81d4988816da11fbcebd0d6 2024-11-12T00:55:28,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/f279f6f236d24fb9b220807f93beafc4 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/f279f6f236d24fb9b220807f93beafc4 2024-11-12T00:55:28,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/6cacdc7c7d014b19998c6ba06f795411 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/6cacdc7c7d014b19998c6ba06f795411 2024-11-12T00:55:28,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/91481ab0f2de4059a732d84dc50009d2 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/91481ab0f2de4059a732d84dc50009d2 2024-11-12T00:55:28,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/bbe2436b2aa44b979bebe66de633f228 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/bbe2436b2aa44b979bebe66de633f228 2024-11-12T00:55:28,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/7e9f1f40084a4ac2a2f4fbfaffb1e3ce to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/7e9f1f40084a4ac2a2f4fbfaffb1e3ce 2024-11-12T00:55:28,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/aeeb4175d70d4cbeb56162be2df1624d to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/aeeb4175d70d4cbeb56162be2df1624d 2024-11-12T00:55:28,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/8b65f7a267574d55b2469761229a9c7b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/8b65f7a267574d55b2469761229a9c7b 2024-11-12T00:55:28,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/c3c06a1cb8c64632abcaa11933793ce2 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/c3c06a1cb8c64632abcaa11933793ce2 2024-11-12T00:55:28,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/0d615d75296f4b88b9ea2d8bfc6faea6 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/0d615d75296f4b88b9ea2d8bfc6faea6 2024-11-12T00:55:28,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/73ef3893f4454e81bbe002ec272c0f26 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/73ef3893f4454e81bbe002ec272c0f26 2024-11-12T00:55:28,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/d82754ba30f346d393f4cd67477c3e43 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/d82754ba30f346d393f4cd67477c3e43 2024-11-12T00:55:28,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/4b6c14ec32fc464ebe850a42fd74ff0c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/4b6c14ec32fc464ebe850a42fd74ff0c 2024-11-12T00:55:28,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/90afa8f6a1c64afb904e3f3cdd790279 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/90afa8f6a1c64afb904e3f3cdd790279 2024-11-12T00:55:28,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/2aaecbff46ec4fbea9aba124183374df to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/2aaecbff46ec4fbea9aba124183374df 2024-11-12T00:55:28,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/89198a6270c1455f8527b6926ab617e4 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/89198a6270c1455f8527b6926ab617e4 2024-11-12T00:55:28,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/9f7bfb870d694d1bbd8e017f5bd49e53 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/9f7bfb870d694d1bbd8e017f5bd49e53 2024-11-12T00:55:28,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/0bb0e61d02e24c71a07aff825b688e79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/0bb0e61d02e24c71a07aff825b688e79 2024-11-12T00:55:28,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/7a29ba6453f54d4c876756a50fbb6375 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/7a29ba6453f54d4c876756a50fbb6375 2024-11-12T00:55:28,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/222224ae470042baa3603d089b84534c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/222224ae470042baa3603d089b84534c 2024-11-12T00:55:28,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/cadc6e856d5b4bcba54005bad7dcc711 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/cadc6e856d5b4bcba54005bad7dcc711 2024-11-12T00:55:28,589 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/recovered.edits/418.seqid, newMaxSeqId=418, maxSeqId=1 2024-11-12T00:55:28,590 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. 2024-11-12T00:55:28,590 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1676): Region close journal for 2400f2d4915c3c25d637e8ed29b4fadd: Waiting for close lock at 1731372927220Running coprocessor pre-close hooks at 1731372927220Disabling compacts and flushes for region at 1731372927220Disabling writes for close at 1731372927220Obtaining lock to block concurrent updates at 1731372927221 (+1 ms)Preparing flush snapshotting stores in 2400f2d4915c3c25d637e8ed29b4fadd at 1731372927221Finished memstore snapshotting TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731372927222 (+1 ms)Flushing stores of TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd. at 1731372927223 (+1 ms)Flushing 2400f2d4915c3c25d637e8ed29b4fadd/A: creating writer at 1731372927223Flushing 2400f2d4915c3c25d637e8ed29b4fadd/A: appending metadata at 1731372927229 (+6 ms)Flushing 2400f2d4915c3c25d637e8ed29b4fadd/A: closing flushed file at 1731372927229Flushing 2400f2d4915c3c25d637e8ed29b4fadd/B: creating writer at 1731372927643 (+414 ms)Flushing 2400f2d4915c3c25d637e8ed29b4fadd/B: appending metadata at 1731372927648 (+5 ms)Flushing 2400f2d4915c3c25d637e8ed29b4fadd/B: closing flushed file at 1731372927648Flushing 2400f2d4915c3c25d637e8ed29b4fadd/C: creating writer at 1731372928065 (+417 ms)Flushing 2400f2d4915c3c25d637e8ed29b4fadd/C: appending metadata at 1731372928068 (+3 ms)Flushing 2400f2d4915c3c25d637e8ed29b4fadd/C: closing flushed file at 1731372928068Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54ce7c88: reopening flushed file at 1731372928480 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6280bbd3: reopening flushed file at 1731372928488 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dbe9b49: reopening flushed file at 1731372928491 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 2400f2d4915c3c25d637e8ed29b4fadd in 1275ms, sequenceid=415, compaction requested=true at 1731372928496 (+5 ms)Writing region close event to WAL at 1731372928586 (+90 ms)Running coprocessor post-close hooks at 1731372928589 (+3 ms)Closed at 1731372928589 2024-11-12T00:55:28,592 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(157): Closed 2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:28,592 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=2400f2d4915c3c25d637e8ed29b4fadd, regionState=CLOSED 2024-11-12T00:55:28,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:55:28,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-12T00:55:28,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseRegionProcedure 2400f2d4915c3c25d637e8ed29b4fadd, server=6928b69dca1b,33953,1731372765559 in 1.5290 sec 2024-11-12T00:55:28,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-12T00:55:28,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2400f2d4915c3c25d637e8ed29b4fadd, UNASSIGN in 1.5330 sec 2024-11-12T00:55:28,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-12T00:55:28,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5370 sec 2024-11-12T00:55:28,604 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372928604"}]},"ts":"1731372928604"} 2024-11-12T00:55:28,605 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T00:55:28,605 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T00:55:28,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5530 sec 2024-11-12T00:55:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T00:55:29,206 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:29,208 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T00:55:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:29,211 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T00:55:29,213 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=131, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:29,217 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:29,220 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/recovered.edits] 2024-11-12T00:55:29,225 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/d9d6e14ffd074cdeb6414f92da028c2a to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/d9d6e14ffd074cdeb6414f92da028c2a 2024-11-12T00:55:29,227 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/de0ed0829c6e4386bc9b2ef635cc6c3c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/de0ed0829c6e4386bc9b2ef635cc6c3c 2024-11-12T00:55:29,229 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/f6c5ad4338d2412b9fa0fe2c3242eddc to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/A/f6c5ad4338d2412b9fa0fe2c3242eddc 2024-11-12T00:55:29,234 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/04bc17a4ee4741a4b5446739c3543643 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/04bc17a4ee4741a4b5446739c3543643 2024-11-12T00:55:29,236 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/294d3082e8fc49af81fbc2ad3cf03f00 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/294d3082e8fc49af81fbc2ad3cf03f00 2024-11-12T00:55:29,238 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/9f7d1cdd1c854d628b4af477bf667008 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/B/9f7d1cdd1c854d628b4af477bf667008 2024-11-12T00:55:29,243 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/145c62a9586d466592167268b756f45e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/145c62a9586d466592167268b756f45e 2024-11-12T00:55:29,246 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/2403f2262666476dbe5adad2c8574f2d to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/2403f2262666476dbe5adad2c8574f2d 2024-11-12T00:55:29,248 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/d4b2965e4f974641a18097abcddab1a4 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/C/d4b2965e4f974641a18097abcddab1a4 2024-11-12T00:55:29,253 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/recovered.edits/418.seqid to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd/recovered.edits/418.seqid 2024-11-12T00:55:29,254 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/2400f2d4915c3c25d637e8ed29b4fadd 2024-11-12T00:55:29,254 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T00:55:29,258 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=131, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:29,262 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T00:55:29,265 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T00:55:29,266 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=131, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:29,266 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T00:55:29,267 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731372929266"}]},"ts":"9223372036854775807"} 2024-11-12T00:55:29,269 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T00:55:29,269 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2400f2d4915c3c25d637e8ed29b4fadd, NAME => 'TestAcidGuarantees,,1731372896423.2400f2d4915c3c25d637e8ed29b4fadd.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T00:55:29,269 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T00:55:29,269 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731372929269"}]},"ts":"9223372036854775807"} 2024-11-12T00:55:29,271 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T00:55:29,272 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=131, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:29,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 64 msec 2024-11-12T00:55:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T00:55:29,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T00:55:29,320 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:29,330 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247 (was 249), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=352 (was 385), ProcessCount=11 (was 11), AvailableMemoryMB=5056 (was 5101) 2024-11-12T00:55:29,341 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=352, ProcessCount=11, AvailableMemoryMB=5056 2024-11-12T00:55:29,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T00:55:29,342 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T00:55:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:29,345 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T00:55:29,345 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:29,345 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 132 2024-11-12T00:55:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-12T00:55:29,346 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T00:55:29,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742356_1532 (size=1061) 2024-11-12T00:55:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-12T00:55:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-12T00:55:29,757 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6 2024-11-12T00:55:29,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742357_1533 (size=53) 2024-11-12T00:55:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-12T00:55:30,167 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:55:30,168 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 891b430dc04c2e04404d2006f883443e, disabling compactions & flushes 2024-11-12T00:55:30,168 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:30,168 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:30,168 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. after waiting 0 ms 2024-11-12T00:55:30,168 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:30,168 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:30,168 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 891b430dc04c2e04404d2006f883443e: Waiting for close lock at 1731372930168Disabling compacts and flushes for region at 1731372930168Disabling writes for close at 1731372930168Writing region close event to WAL at 1731372930168Closed at 1731372930168 2024-11-12T00:55:30,170 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T00:55:30,170 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731372930170"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731372930170"}]},"ts":"1731372930170"} 2024-11-12T00:55:30,173 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T00:55:30,174 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T00:55:30,175 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372930174"}]},"ts":"1731372930174"} 2024-11-12T00:55:30,177 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T00:55:30,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=891b430dc04c2e04404d2006f883443e, ASSIGN}] 2024-11-12T00:55:30,178 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=891b430dc04c2e04404d2006f883443e, ASSIGN 2024-11-12T00:55:30,179 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=891b430dc04c2e04404d2006f883443e, ASSIGN; state=OFFLINE, location=6928b69dca1b,33953,1731372765559; forceNewPlan=false, retain=false 2024-11-12T00:55:30,330 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=891b430dc04c2e04404d2006f883443e, regionState=OPENING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:30,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=891b430dc04c2e04404d2006f883443e, ASSIGN because future has completed 2024-11-12T00:55:30,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure 891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:55:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-12T00:55:30,491 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:30,491 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7752): Opening region: {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} 2024-11-12T00:55:30,491 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:30,491 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:55:30,492 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7794): checking encryption for 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:30,492 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7797): checking classloading for 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:30,493 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:30,495 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:55:30,495 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 891b430dc04c2e04404d2006f883443e columnFamilyName A 2024-11-12T00:55:30,495 DEBUG [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:30,495 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.HStore(327): Store=891b430dc04c2e04404d2006f883443e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:55:30,495 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:30,497 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:55:30,497 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 891b430dc04c2e04404d2006f883443e columnFamilyName B 2024-11-12T00:55:30,497 DEBUG [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:30,497 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.HStore(327): Store=891b430dc04c2e04404d2006f883443e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:55:30,497 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:30,498 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:55:30,499 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 891b430dc04c2e04404d2006f883443e columnFamilyName C 2024-11-12T00:55:30,499 DEBUG [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:30,499 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.HStore(327): Store=891b430dc04c2e04404d2006f883443e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:55:30,499 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:30,499 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1038): replaying wal for 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:30,500 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:30,500 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:30,502 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1048): stopping wal replay for 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:30,502 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1060): Cleaning up temporary data for 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:30,502 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T00:55:30,504 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1093): writing seq id for 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:30,507 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T00:55:30,508 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1114): Opened 891b430dc04c2e04404d2006f883443e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68058971, jitterRate=0.014157697558403015}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T00:55:30,508 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:30,509 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1006): Region open journal for 891b430dc04c2e04404d2006f883443e: Running coprocessor pre-open hook at 1731372930492Writing region info on filesystem at 1731372930492Initializing all the Stores at 1731372930493 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372930493Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372930493Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372930493Cleaning up temporary data from old regions at 1731372930502 (+9 ms)Running coprocessor post-open hooks at 1731372930508 (+6 ms)Region opened successfully at 1731372930508 2024-11-12T00:55:30,509 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., pid=134, masterSystemTime=1731372930484 2024-11-12T00:55:30,512 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:30,512 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:30,513 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=891b430dc04c2e04404d2006f883443e, regionState=OPEN, openSeqNum=2, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:30,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure 891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:55:30,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-12T00:55:30,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; OpenRegionProcedure 891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 in 184 msec 2024-11-12T00:55:30,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-12T00:55:30,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=891b430dc04c2e04404d2006f883443e, ASSIGN in 342 msec 2024-11-12T00:55:30,523 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T00:55:30,523 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372930523"}]},"ts":"1731372930523"} 2024-11-12T00:55:30,525 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T00:55:30,526 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T00:55:30,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1830 sec 2024-11-12T00:55:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-12T00:55:31,496 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:31,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a769332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:31,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:55:31,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:55:31,499 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:55:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:55:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:55:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bafdca8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:31,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:55:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:55:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:31,502 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:55:31,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e3ecf10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:31,504 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:55:31,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:55:31,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:55:31,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:55:31,508 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T00:55:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 6928b69dca1b,45925,1731372764943 2024-11-12T00:55:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b2caab5 2024-11-12T00:55:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T00:55:31,511 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T00:55:31,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T00:55:31,513 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T00:55:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:31,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742358_1534 (size=1097) 2024-11-12T00:55:31,930 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T00:55:31,930 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T00:55:31,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T00:55:31,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=891b430dc04c2e04404d2006f883443e, REOPEN/MOVE}] 2024-11-12T00:55:31,939 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=891b430dc04c2e04404d2006f883443e, REOPEN/MOVE 2024-11-12T00:55:31,940 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=891b430dc04c2e04404d2006f883443e, regionState=CLOSING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:31,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=891b430dc04c2e04404d2006f883443e, REOPEN/MOVE because future has completed 2024-11-12T00:55:31,943 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T00:55:31,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:55:32,095 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,095 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T00:55:32,096 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 891b430dc04c2e04404d2006f883443e, disabling compactions & flushes 2024-11-12T00:55:32,096 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:32,096 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:32,096 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. after waiting 0 ms 2024-11-12T00:55:32,096 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:32,105 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T00:55:32,106 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:32,106 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 891b430dc04c2e04404d2006f883443e: Waiting for close lock at 1731372932096Running coprocessor pre-close hooks at 1731372932096Disabling compacts and flushes for region at 1731372932096Disabling writes for close at 1731372932096Writing region close event to WAL at 1731372932097 (+1 ms)Running coprocessor post-close hooks at 1731372932106 (+9 ms)Closed at 1731372932106 2024-11-12T00:55:32,106 WARN [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionServer(3299): Not adding moved region record: 891b430dc04c2e04404d2006f883443e to self. 2024-11-12T00:55:32,109 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,109 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=891b430dc04c2e04404d2006f883443e, regionState=CLOSED 2024-11-12T00:55:32,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:55:32,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-12T00:55:32,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 in 169 msec 2024-11-12T00:55:32,116 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=891b430dc04c2e04404d2006f883443e, REOPEN/MOVE; state=CLOSED, location=6928b69dca1b,33953,1731372765559; forceNewPlan=false, retain=true 2024-11-12T00:55:32,268 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=891b430dc04c2e04404d2006f883443e, regionState=OPENING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,272 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=891b430dc04c2e04404d2006f883443e, REOPEN/MOVE because future has completed 2024-11-12T00:55:32,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:55:32,433 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:32,434 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7752): Opening region: {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} 2024-11-12T00:55:32,434 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,435 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T00:55:32,435 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7794): checking encryption for 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,435 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7797): checking classloading for 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,438 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,439 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:55:32,439 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 891b430dc04c2e04404d2006f883443e columnFamilyName A 2024-11-12T00:55:32,441 DEBUG [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:32,441 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.HStore(327): Store=891b430dc04c2e04404d2006f883443e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:55:32,442 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,442 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:55:32,443 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 891b430dc04c2e04404d2006f883443e columnFamilyName B 2024-11-12T00:55:32,443 DEBUG [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:32,443 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.HStore(327): Store=891b430dc04c2e04404d2006f883443e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:55:32,443 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,444 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T00:55:32,444 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 891b430dc04c2e04404d2006f883443e columnFamilyName C 2024-11-12T00:55:32,444 DEBUG [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:32,445 INFO [StoreOpener-891b430dc04c2e04404d2006f883443e-1 {}] regionserver.HStore(327): Store=891b430dc04c2e04404d2006f883443e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T00:55:32,445 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:32,445 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1038): replaying wal for 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,446 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,447 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,449 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1048): stopping wal replay for 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,449 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1060): Cleaning up temporary data for 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,449 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T00:55:32,451 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1093): writing seq id for 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,452 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1114): Opened 891b430dc04c2e04404d2006f883443e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61763218, jitterRate=-0.07965633273124695}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T00:55:32,452 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,452 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1006): Region open journal for 891b430dc04c2e04404d2006f883443e: Running coprocessor pre-open hook at 1731372932435Writing region info on filesystem at 1731372932435Initializing all the Stores at 1731372932437 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372932437Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372932438 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731372932438Cleaning up temporary data from old regions at 1731372932449 (+11 ms)Running coprocessor post-open hooks at 1731372932452 (+3 ms)Region opened successfully at 1731372932452 2024-11-12T00:55:32,453 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., pid=139, masterSystemTime=1731372932426 2024-11-12T00:55:32,456 DEBUG [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:32,456 INFO [RS_OPEN_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:32,457 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=891b430dc04c2e04404d2006f883443e, regionState=OPEN, openSeqNum=5, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:55:32,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=137 2024-11-12T00:55:32,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 in 188 msec 2024-11-12T00:55:32,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-12T00:55:32,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=891b430dc04c2e04404d2006f883443e, REOPEN/MOVE in 524 msec 2024-11-12T00:55:32,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-12T00:55:32,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 532 msec 2024-11-12T00:55:32,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 956 msec 2024-11-12T00:55:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T00:55:32,472 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:32,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd32fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,473 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:55:32,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:55:32,475 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:55:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:55:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:55:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7239f61a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:55:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:55:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:32,477 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:55:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0a21f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,479 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:55:32,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:55:32,480 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:55:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:55:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:55:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441a62e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:55:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:55:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:32,481 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:55:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23c84447, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,483 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:55:32,483 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:55:32,484 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:55:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:55:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:55:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20bff063, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:55:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:55:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:32,486 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:55:32,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@648b3bbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,487 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:55:32,487 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:55:32,488 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:55:32,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:55:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:55:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@798c573c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:55:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:55:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:32,490 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:55:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6c21cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:55:32,492 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:55:32,492 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:55:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:55:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:55:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1df7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:55:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:55:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:32,494 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:55:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5885acea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,496 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:55:32,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:55:32,497 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:55:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:55:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:55:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1c97d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:55:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:55:32,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:32,498 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:55:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aa9a294, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,500 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:55:32,500 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:55:32,501 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:55:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:55:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:55:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a32b20a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:55:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:55:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:32,503 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:55:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67982d69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,504 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:55:32,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:55:32,505 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:55:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:55:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:55:32,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f4fbfb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:55:32,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:55:32,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:32,507 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:55:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@627be0af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,508 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,508 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:55:32,508 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:55:32,509 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:55:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:55:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:55:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62c75921, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:55:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:55:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:32,510 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:55:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2f10af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,511 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6928b69dca1b,45925,-1 for getting cluster id 2024-11-12T00:55:32,511 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T00:55:32,512 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdc4e098-f842-461c-a9b9-51c8c13368ac' 2024-11-12T00:55:32,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T00:55:32,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdc4e098-f842-461c-a9b9-51c8c13368ac" 2024-11-12T00:55:32,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a82f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6928b69dca1b,45925,-1] 2024-11-12T00:55:32,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T00:55:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:32,513 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T00:55:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@568eaef1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T00:55:32,514 DEBUG [Thread-2406 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:55:32,514 DEBUG [Thread-2408 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:55:32,515 DEBUG [Thread-2399 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:55:32,515 DEBUG [Thread-2391 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:55:32,515 DEBUG [Thread-2397 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:55:32,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:55:32,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:55:32,515 DEBUG [Thread-2393 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:55:32,515 DEBUG [Thread-2402 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:55:32,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:55:32,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:55:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:55:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:55:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:55:32,516 DEBUG [Thread-2395 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:55:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:55:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:55:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:55:32,516 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:55:32,516 DEBUG [Thread-2404 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:55:32,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:55:32,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:55:32,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:55:32,517 DEBUG [Thread-2410 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T00:55:32,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:55:32,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:55:32,517 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:55:32,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:55:32,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:55:32,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:55:32,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:55:32,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:55:32,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6928b69dca1b,33953,1731372765559, seqNum=-1] 2024-11-12T00:55:32,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:55:32,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:55:32,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T00:55:32,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:55:32,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:55:32,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:55:32,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:55:32,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:55:32,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T00:55:32,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:55:32,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:55:32,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:55:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-12T00:55:32,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:55:32,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:55:32,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:55:32,521 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:55:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-12T00:55:32,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:55:32,522 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:55:32,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:55:32,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:55:32,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5] 2024-11-12T00:55:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T00:55:32,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:32,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:32,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:32,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:32,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:32,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:32,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ee7d257ed77b4c1a95266b67c623ffd8_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372932527/Put/seqid=0 2024-11-12T00:55:32,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1731372932542, max timestamp=1731372932551Num uniques 859; 2024-11-12T00:55:32,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:55:32,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731372932542, max timestamp=1731372932551Num uniques 859; 2024-11-12T00:55:32,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-12T00:55:32,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731372932542, max timestamp=1731372932551Num uniques 859; 2024-11-12T00:55:32,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-12T00:55:32,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372992567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742359_1535 (size=17034) 2024-11-12T00:55:32,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:32,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372992570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,575 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:32,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372992571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:32,578 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ee7d257ed77b4c1a95266b67c623ffd8_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ee7d257ed77b4c1a95266b67c623ffd8_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:32,579 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/de6e8b0ec4634bf4a1c4dc9389050144, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:32,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:32,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/de6e8b0ec4634bf4a1c4dc9389050144 is 175, key is test_row_0/A:col10/1731372932527/Put/seqid=0 2024-11-12T00:55:32,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372992572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:32,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372992574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:32,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742360_1536 (size=48139) 2024-11-12T00:55:32,593 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/de6e8b0ec4634bf4a1c4dc9389050144 2024-11-12T00:55:32,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/aa5b07816228444dadad8f78f5b58214 is 50, key is test_row_0/B:col10/1731372932527/Put/seqid=0 2024-11-12T00:55:32,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742361_1537 (size=12001) 2024-11-12T00:55:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-12T00:55:32,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-12T00:55:32,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:32,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:32,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:32,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:32,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:32,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:32,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372992685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:32,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372992685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:32,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:32,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372992685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372992685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372992685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:32,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-12T00:55:32,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:32,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:32,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:32,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:32,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-12T00:55:32,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372992895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372992895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:32,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372992895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:32,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372992896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372992896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:32,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-12T00:55:32,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:32,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:32,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:32,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:32,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:33,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/aa5b07816228444dadad8f78f5b58214 2024-11-12T00:55:33,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/f3fb9bb36375479188fff124eb667a92 is 50, key is test_row_0/C:col10/1731372932527/Put/seqid=0 2024-11-12T00:55:33,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742362_1538 (size=12001) 2024-11-12T00:55:33,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-12T00:55:33,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:33,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:33,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:33,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:33,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:33,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-12T00:55:33,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372993205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:33,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372993205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372993205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372993205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:33,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372993205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:33,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-12T00:55:33,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:33,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:33,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:33,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:33,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:33,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:33,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-12T00:55:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:33,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:33,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:33,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/f3fb9bb36375479188fff124eb667a92 2024-11-12T00:55:33,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/de6e8b0ec4634bf4a1c4dc9389050144 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/de6e8b0ec4634bf4a1c4dc9389050144 2024-11-12T00:55:33,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/de6e8b0ec4634bf4a1c4dc9389050144, entries=250, sequenceid=16, filesize=47.0 K 2024-11-12T00:55:33,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/aa5b07816228444dadad8f78f5b58214 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/aa5b07816228444dadad8f78f5b58214 2024-11-12T00:55:33,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/aa5b07816228444dadad8f78f5b58214, entries=150, sequenceid=16, filesize=11.7 K 2024-11-12T00:55:33,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/f3fb9bb36375479188fff124eb667a92 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/f3fb9bb36375479188fff124eb667a92 2024-11-12T00:55:33,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/f3fb9bb36375479188fff124eb667a92, entries=150, sequenceid=16, filesize=11.7 K 2024-11-12T00:55:33,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=122.23 KB/125162 for 891b430dc04c2e04404d2006f883443e in 936ms, sequenceid=16, compaction requested=false 2024-11-12T00:55:33,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:33,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-12T00:55:33,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:33,592 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=122.23 KB heapSize=348.39 KB 2024-11-12T00:55:33,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:33,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:33,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:33,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:33,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:33,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:33,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112cd5322b389d746c295021053dba200e0_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372932564/Put/seqid=0 2024-11-12T00:55:33,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742363_1539 (size=12154) 2024-11-12T00:55:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-12T00:55:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:33,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:33,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372993724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:33,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:33,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:33,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372993725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372993725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:33,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:33,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:33,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:33,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372993726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:33,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372993726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:33,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:33,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:33,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:33,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:33,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372993836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:33,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372993836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:33,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372993836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372993836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:33,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372993836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:34,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:34,017 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112cd5322b389d746c295021053dba200e0_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112cd5322b389d746c295021053dba200e0_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:34,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/0f33a90008154adfa5356b595f2cca3d, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:34,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/0f33a90008154adfa5356b595f2cca3d is 175, key is test_row_0/A:col10/1731372932564/Put/seqid=0 2024-11-12T00:55:34,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742364_1540 (size=30955) 2024-11-12T00:55:34,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372994045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372994046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:34,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372994046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:34,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372994046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:34,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372994046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:34,107 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T00:55:34,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372994355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:34,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372994355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372994355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:34,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372994355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372994356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:34,421 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/0f33a90008154adfa5356b595f2cca3d 2024-11-12T00:55:34,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/7f09d0a5002e4798973e987c4abd5088 is 50, key is test_row_0/B:col10/1731372932564/Put/seqid=0 2024-11-12T00:55:34,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742365_1541 (size=12001) 2024-11-12T00:55:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-12T00:55:34,831 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.87 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/7f09d0a5002e4798973e987c4abd5088 2024-11-12T00:55:34,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/11f747931a42466fa9a9b98fa5a2909f is 50, key is test_row_0/C:col10/1731372932564/Put/seqid=0 2024-11-12T00:55:34,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742366_1542 (size=12001) 2024-11-12T00:55:34,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372994865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:34,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372994865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372994866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372994866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:34,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372994866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:35,240 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.87 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/11f747931a42466fa9a9b98fa5a2909f 2024-11-12T00:55:35,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/0f33a90008154adfa5356b595f2cca3d as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/0f33a90008154adfa5356b595f2cca3d 2024-11-12T00:55:35,246 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/0f33a90008154adfa5356b595f2cca3d, entries=150, sequenceid=51, filesize=30.2 K 2024-11-12T00:55:35,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/7f09d0a5002e4798973e987c4abd5088 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/7f09d0a5002e4798973e987c4abd5088 2024-11-12T00:55:35,249 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/7f09d0a5002e4798973e987c4abd5088, entries=150, sequenceid=51, filesize=11.7 K 2024-11-12T00:55:35,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/11f747931a42466fa9a9b98fa5a2909f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/11f747931a42466fa9a9b98fa5a2909f 2024-11-12T00:55:35,258 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/11f747931a42466fa9a9b98fa5a2909f, entries=150, sequenceid=51, filesize=11.7 K 2024-11-12T00:55:35,259 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~122.23 KB/125162, heapSize ~348.34 KB/356704, currentSize=67.09 KB/68700 for 891b430dc04c2e04404d2006f883443e in 1668ms, sequenceid=51, compaction requested=false 2024-11-12T00:55:35,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:35,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:35,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-12T00:55:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-12T00:55:35,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-12T00:55:35,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7380 sec 2024-11-12T00:55:35,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 2.7450 sec 2024-11-12T00:55:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:35,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T00:55:35,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:35,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:35,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:35,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:35,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:35,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:35,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125904766eda3345fa943c83e0a0ee0c62_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372935877/Put/seqid=0 2024-11-12T00:55:35,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742367_1543 (size=17034) 2024-11-12T00:55:35,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372995895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:35,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:35,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372995897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:35,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:35,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:35,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:35,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372995897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:35,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:35,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372995897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:35,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372995898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:35,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372996005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:36,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372996006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372996006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372996006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:36,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372996006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372996216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372996216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372996216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372996216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:36,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372996216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,287 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:36,290 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125904766eda3345fa943c83e0a0ee0c62_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125904766eda3345fa943c83e0a0ee0c62_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:36,291 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/1375d4b548d745f298b259b52854f0e4, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:36,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/1375d4b548d745f298b259b52854f0e4 is 175, key is test_row_0/A:col10/1731372935877/Put/seqid=0 2024-11-12T00:55:36,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742368_1544 (size=48139) 2024-11-12T00:55:36,300 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=66, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/1375d4b548d745f298b259b52854f0e4 2024-11-12T00:55:36,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/9ed0ea5784614f8e9f2fd1dd476773dd is 50, key is test_row_0/B:col10/1731372935877/Put/seqid=0 2024-11-12T00:55:36,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742369_1545 (size=12001) 2024-11-12T00:55:36,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372996525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:36,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372996525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372996526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:36,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372996526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372996526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-12T00:55:36,675 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:36,676 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:55:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-12T00:55:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-12T00:55:36,677 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:55:36,678 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:55:36,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:55:36,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/9ed0ea5784614f8e9f2fd1dd476773dd 2024-11-12T00:55:36,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/b3ec7a2910624487849031d2b3994038 is 50, key is test_row_0/C:col10/1731372935877/Put/seqid=0 2024-11-12T00:55:36,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742370_1546 (size=12001) 2024-11-12T00:55:36,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/b3ec7a2910624487849031d2b3994038 2024-11-12T00:55:36,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/1375d4b548d745f298b259b52854f0e4 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/1375d4b548d745f298b259b52854f0e4 2024-11-12T00:55:36,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/1375d4b548d745f298b259b52854f0e4, entries=250, sequenceid=66, filesize=47.0 K 2024-11-12T00:55:36,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/9ed0ea5784614f8e9f2fd1dd476773dd as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/9ed0ea5784614f8e9f2fd1dd476773dd 2024-11-12T00:55:36,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/9ed0ea5784614f8e9f2fd1dd476773dd, entries=150, sequenceid=66, filesize=11.7 K 2024-11-12T00:55:36,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/b3ec7a2910624487849031d2b3994038 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b3ec7a2910624487849031d2b3994038 2024-11-12T00:55:36,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b3ec7a2910624487849031d2b3994038, entries=150, sequenceid=66, filesize=11.7 K 2024-11-12T00:55:36,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 891b430dc04c2e04404d2006f883443e in 854ms, sequenceid=66, compaction requested=true 2024-11-12T00:55:36,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:36,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:55:36,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:36,732 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:36,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:55:36,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:36,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:55:36,732 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:36,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:36,737 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:36,737 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:36,737 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/A is initiating minor compaction (all files) 2024-11-12T00:55:36,737 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/B is initiating minor compaction (all files) 2024-11-12T00:55:36,737 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/B in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:36,737 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/A in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:36,737 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/aa5b07816228444dadad8f78f5b58214, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/7f09d0a5002e4798973e987c4abd5088, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/9ed0ea5784614f8e9f2fd1dd476773dd] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=35.2 K 2024-11-12T00:55:36,737 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/de6e8b0ec4634bf4a1c4dc9389050144, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/0f33a90008154adfa5356b595f2cca3d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/1375d4b548d745f298b259b52854f0e4] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=124.3 K 2024-11-12T00:55:36,737 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:36,737 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/de6e8b0ec4634bf4a1c4dc9389050144, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/0f33a90008154adfa5356b595f2cca3d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/1375d4b548d745f298b259b52854f0e4] 2024-11-12T00:55:36,738 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting de6e8b0ec4634bf4a1c4dc9389050144, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731372932524 2024-11-12T00:55:36,738 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting aa5b07816228444dadad8f78f5b58214, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731372932526 2024-11-12T00:55:36,738 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f09d0a5002e4798973e987c4abd5088, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731372932564 2024-11-12T00:55:36,738 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f33a90008154adfa5356b595f2cca3d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731372932564 2024-11-12T00:55:36,738 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ed0ea5784614f8e9f2fd1dd476773dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731372933724 2024-11-12T00:55:36,738 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1375d4b548d745f298b259b52854f0e4, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731372933723 2024-11-12T00:55:36,744 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:36,746 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:36,746 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/d3fbe87c6d394450bbc2233d83058857 is 50, key is test_row_0/B:col10/1731372935877/Put/seqid=0 2024-11-12T00:55:36,747 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411127429474a1aee4ff786378f656a32aa50_891b430dc04c2e04404d2006f883443e store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:36,750 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411127429474a1aee4ff786378f656a32aa50_891b430dc04c2e04404d2006f883443e, store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:36,750 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127429474a1aee4ff786378f656a32aa50_891b430dc04c2e04404d2006f883443e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:36,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742371_1547 (size=12104) 2024-11-12T00:55:36,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742372_1548 (size=4469) 2024-11-12T00:55:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-12T00:55:36,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-12T00:55:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:36,830 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T00:55:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:36,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c68ea44b40ba42e49a178713724745bb_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372935893/Put/seqid=0 2024-11-12T00:55:36,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742373_1549 (size=12154) 2024-11-12T00:55:36,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-12T00:55:37,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:37,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:37,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372997047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372997047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372997050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372997053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372997053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,154 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#A#compaction#469 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:37,154 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/74909b6470004c129717c9cc5650b957 is 175, key is test_row_0/A:col10/1731372935877/Put/seqid=0 2024-11-12T00:55:37,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372997155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372997155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,157 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/d3fbe87c6d394450bbc2233d83058857 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/d3fbe87c6d394450bbc2233d83058857 2024-11-12T00:55:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742374_1550 (size=31058) 2024-11-12T00:55:37,164 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/B of 891b430dc04c2e04404d2006f883443e into d3fbe87c6d394450bbc2233d83058857(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:37,164 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:37,164 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/B, priority=13, startTime=1731372936732; duration=0sec 2024-11-12T00:55:37,164 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:37,164 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:B 2024-11-12T00:55:37,164 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:37,164 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/74909b6470004c129717c9cc5650b957 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/74909b6470004c129717c9cc5650b957 2024-11-12T00:55:37,165 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:37,165 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/C is initiating minor compaction (all files) 2024-11-12T00:55:37,165 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/C in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:37,165 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/f3fb9bb36375479188fff124eb667a92, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/11f747931a42466fa9a9b98fa5a2909f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b3ec7a2910624487849031d2b3994038] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=35.2 K 2024-11-12T00:55:37,166 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting f3fb9bb36375479188fff124eb667a92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731372932526 2024-11-12T00:55:37,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372997166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,167 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 11f747931a42466fa9a9b98fa5a2909f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731372932564 2024-11-12T00:55:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,167 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting b3ec7a2910624487849031d2b3994038, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731372933724 2024-11-12T00:55:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372997166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372997166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,169 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/A of 891b430dc04c2e04404d2006f883443e into 74909b6470004c129717c9cc5650b957(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:37,169 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:37,169 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/A, priority=13, startTime=1731372936732; duration=0sec 2024-11-12T00:55:37,169 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:37,169 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:A 2024-11-12T00:55:37,173 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#C#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:37,173 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/3d27f65a8ed64410ac9d71e1dd030bd7 is 50, key is test_row_0/C:col10/1731372935877/Put/seqid=0 2024-11-12T00:55:37,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742375_1551 (size=12104) 2024-11-12T00:55:37,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:37,243 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c68ea44b40ba42e49a178713724745bb_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c68ea44b40ba42e49a178713724745bb_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:37,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/e62dc7277f45486bb12a25c6665856c8, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:37,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/e62dc7277f45486bb12a25c6665856c8 is 175, key is test_row_0/A:col10/1731372935893/Put/seqid=0 2024-11-12T00:55:37,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742376_1552 (size=30955) 2024-11-12T00:55:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-12T00:55:37,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372997366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372997366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372997375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372997375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372997375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,583 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/3d27f65a8ed64410ac9d71e1dd030bd7 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/3d27f65a8ed64410ac9d71e1dd030bd7 2024-11-12T00:55:37,586 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/C of 891b430dc04c2e04404d2006f883443e into 3d27f65a8ed64410ac9d71e1dd030bd7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:37,586 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:37,586 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/C, priority=13, startTime=1731372936732; duration=0sec 2024-11-12T00:55:37,586 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:37,586 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:C 2024-11-12T00:55:37,648 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/e62dc7277f45486bb12a25c6665856c8 2024-11-12T00:55:37,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/3c31b77aaa6542399398ee36c1b013a3 is 50, key is test_row_0/B:col10/1731372935893/Put/seqid=0 2024-11-12T00:55:37,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742377_1553 (size=12001) 2024-11-12T00:55:37,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372997675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372997676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372997686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372997686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372997686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:37,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:37,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-12T00:55:38,056 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/3c31b77aaa6542399398ee36c1b013a3 2024-11-12T00:55:38,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/2eb41b74008f43faa1e828b159e94038 is 50, key is test_row_0/C:col10/1731372935893/Put/seqid=0 2024-11-12T00:55:38,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742378_1554 (size=12001) 2024-11-12T00:55:38,065 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/2eb41b74008f43faa1e828b159e94038 2024-11-12T00:55:38,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/e62dc7277f45486bb12a25c6665856c8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/e62dc7277f45486bb12a25c6665856c8 2024-11-12T00:55:38,072 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/e62dc7277f45486bb12a25c6665856c8, entries=150, sequenceid=83, filesize=30.2 K 2024-11-12T00:55:38,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/3c31b77aaa6542399398ee36c1b013a3 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/3c31b77aaa6542399398ee36c1b013a3 2024-11-12T00:55:38,076 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/3c31b77aaa6542399398ee36c1b013a3, entries=150, sequenceid=83, filesize=11.7 K 2024-11-12T00:55:38,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/2eb41b74008f43faa1e828b159e94038 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/2eb41b74008f43faa1e828b159e94038 2024-11-12T00:55:38,079 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/2eb41b74008f43faa1e828b159e94038, entries=150, sequenceid=83, filesize=11.7 K 2024-11-12T00:55:38,080 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 891b430dc04c2e04404d2006f883443e in 1250ms, sequenceid=83, compaction requested=false 2024-11-12T00:55:38,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:38,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:38,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-12T00:55:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-12T00:55:38,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-12T00:55:38,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-11-12T00:55:38,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.4080 sec 2024-11-12T00:55:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:38,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T00:55:38,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:38,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:38,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:38,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:38,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:38,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:38,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a9eaecb81b354ddca3e7ab28bd29a24c_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372938188/Put/seqid=0 2024-11-12T00:55:38,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742379_1555 (size=14594) 2024-11-12T00:55:38,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372998201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372998202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372998203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372998205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372998206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372998316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372998316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372998316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372998316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372998316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372998526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372998526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372998526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372998526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372998526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,600 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:38,603 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a9eaecb81b354ddca3e7ab28bd29a24c_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a9eaecb81b354ddca3e7ab28bd29a24c_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:38,603 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/8fb09c8fe7164c1a985a221f7c983131, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:38,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/8fb09c8fe7164c1a985a221f7c983131 is 175, key is test_row_0/A:col10/1731372938188/Put/seqid=0 2024-11-12T00:55:38,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742380_1556 (size=39549) 2024-11-12T00:55:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-12T00:55:38,836 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:38,836 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:55:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-12T00:55:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T00:55:38,838 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:55:38,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,839 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:55:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372998836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,839 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:55:38,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372998836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372998836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372998836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372998836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T00:55:38,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T00:55:38,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:38,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:38,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:38,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:38,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,009 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/8fb09c8fe7164c1a985a221f7c983131 2024-11-12T00:55:39,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/c6fd46b918b5466e87c7100f80ee273e is 50, key is test_row_0/B:col10/1731372938188/Put/seqid=0 2024-11-12T00:55:39,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742381_1557 (size=12001) 2024-11-12T00:55:39,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T00:55:39,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:39,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:39,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:39,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T00:55:39,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T00:55:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:39,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731372999345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:39,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:39,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731372999345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731372999345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:39,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:39,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731372999346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731372999346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:39,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/c6fd46b918b5466e87c7100f80ee273e 2024-11-12T00:55:39,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/6e84dcfe5f454391a3a2a5cfd05c5b05 is 50, key is test_row_0/C:col10/1731372938188/Put/seqid=0 2024-11-12T00:55:39,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742382_1558 (size=12001) 2024-11-12T00:55:39,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T00:55:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:39,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T00:55:39,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T00:55:39,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:39,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T00:55:39,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:39,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:39,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:39,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:39,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/6e84dcfe5f454391a3a2a5cfd05c5b05 2024-11-12T00:55:39,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/8fb09c8fe7164c1a985a221f7c983131 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/8fb09c8fe7164c1a985a221f7c983131 2024-11-12T00:55:39,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/8fb09c8fe7164c1a985a221f7c983131, entries=200, sequenceid=104, filesize=38.6 K 2024-11-12T00:55:39,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/c6fd46b918b5466e87c7100f80ee273e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c6fd46b918b5466e87c7100f80ee273e 2024-11-12T00:55:39,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c6fd46b918b5466e87c7100f80ee273e, entries=150, sequenceid=104, filesize=11.7 K 2024-11-12T00:55:39,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/6e84dcfe5f454391a3a2a5cfd05c5b05 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/6e84dcfe5f454391a3a2a5cfd05c5b05 2024-11-12T00:55:39,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/6e84dcfe5f454391a3a2a5cfd05c5b05, entries=150, sequenceid=104, filesize=11.7 K 2024-11-12T00:55:39,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 891b430dc04c2e04404d2006f883443e in 1661ms, sequenceid=104, compaction requested=true 2024-11-12T00:55:39,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:39,849 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:39,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:55:39,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:39,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:55:39,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:39,849 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:39,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:55:39,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:39,850 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:39,850 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:39,850 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/A is initiating minor compaction (all files) 2024-11-12T00:55:39,850 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/B is initiating minor compaction (all files) 2024-11-12T00:55:39,850 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/A in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:39,850 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/B in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:39,850 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/74909b6470004c129717c9cc5650b957, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/e62dc7277f45486bb12a25c6665856c8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/8fb09c8fe7164c1a985a221f7c983131] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=99.2 K 2024-11-12T00:55:39,850 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/d3fbe87c6d394450bbc2233d83058857, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/3c31b77aaa6542399398ee36c1b013a3, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c6fd46b918b5466e87c7100f80ee273e] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=35.3 K 2024-11-12T00:55:39,850 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:39,850 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/74909b6470004c129717c9cc5650b957, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/e62dc7277f45486bb12a25c6665856c8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/8fb09c8fe7164c1a985a221f7c983131] 2024-11-12T00:55:39,851 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3fbe87c6d394450bbc2233d83058857, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731372933724 2024-11-12T00:55:39,851 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 74909b6470004c129717c9cc5650b957, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731372933724 2024-11-12T00:55:39,851 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c31b77aaa6542399398ee36c1b013a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731372935893 2024-11-12T00:55:39,852 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting e62dc7277f45486bb12a25c6665856c8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731372935893 2024-11-12T00:55:39,852 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6fd46b918b5466e87c7100f80ee273e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731372937052 2024-11-12T00:55:39,852 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fb09c8fe7164c1a985a221f7c983131, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731372937046 2024-11-12T00:55:39,857 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:39,859 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#B#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:39,859 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112c9ff1cd7fe4646b9abe450a19b41d1e2_891b430dc04c2e04404d2006f883443e store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:39,859 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/41c8d1af63344123bf8f703edc9499d8 is 50, key is test_row_0/B:col10/1731372938188/Put/seqid=0 2024-11-12T00:55:39,861 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112c9ff1cd7fe4646b9abe450a19b41d1e2_891b430dc04c2e04404d2006f883443e, store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:39,862 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c9ff1cd7fe4646b9abe450a19b41d1e2_891b430dc04c2e04404d2006f883443e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:39,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742383_1559 (size=12207) 2024-11-12T00:55:39,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742384_1560 (size=4469) 2024-11-12T00:55:39,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T00:55:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:39,906 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T00:55:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:39,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125260167f525f4bb399fcf8009849086d_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372938202/Put/seqid=0 2024-11-12T00:55:39,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742385_1561 (size=12154) 2024-11-12T00:55:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T00:55:40,265 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#A#compaction#478 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:40,265 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/253a021c104d41fba39dae34d327b40b is 175, key is test_row_0/A:col10/1731372938188/Put/seqid=0 2024-11-12T00:55:40,267 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/41c8d1af63344123bf8f703edc9499d8 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/41c8d1af63344123bf8f703edc9499d8 2024-11-12T00:55:40,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742386_1562 (size=31161) 2024-11-12T00:55:40,271 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/B of 891b430dc04c2e04404d2006f883443e into 41c8d1af63344123bf8f703edc9499d8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:40,271 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:40,271 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/B, priority=13, startTime=1731372939849; duration=0sec 2024-11-12T00:55:40,271 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:40,271 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:B 2024-11-12T00:55:40,271 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:40,272 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:40,272 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/C is initiating minor compaction (all files) 2024-11-12T00:55:40,272 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/C in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:40,272 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/3d27f65a8ed64410ac9d71e1dd030bd7, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/2eb41b74008f43faa1e828b159e94038, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/6e84dcfe5f454391a3a2a5cfd05c5b05] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=35.3 K 2024-11-12T00:55:40,272 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d27f65a8ed64410ac9d71e1dd030bd7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731372933724 2024-11-12T00:55:40,273 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eb41b74008f43faa1e828b159e94038, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731372935893 2024-11-12T00:55:40,273 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e84dcfe5f454391a3a2a5cfd05c5b05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731372937052 2024-11-12T00:55:40,282 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#C#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:40,282 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/4779a91b267d42c694919c4e598d7898 is 50, key is test_row_0/C:col10/1731372938188/Put/seqid=0 2024-11-12T00:55:40,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742387_1563 (size=12207) 2024-11-12T00:55:40,290 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/4779a91b267d42c694919c4e598d7898 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/4779a91b267d42c694919c4e598d7898 2024-11-12T00:55:40,294 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/C of 891b430dc04c2e04404d2006f883443e into 4779a91b267d42c694919c4e598d7898(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:40,294 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:40,294 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/C, priority=13, startTime=1731372939849; duration=0sec 2024-11-12T00:55:40,294 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:40,294 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:C 2024-11-12T00:55:40,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:40,317 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125260167f525f4bb399fcf8009849086d_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125260167f525f4bb399fcf8009849086d_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:40,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/fd0600e33b3d42019556600022fe08a5, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:40,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/fd0600e33b3d42019556600022fe08a5 is 175, key is test_row_0/A:col10/1731372938202/Put/seqid=0 2024-11-12T00:55:40,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742388_1564 (size=30955) 2024-11-12T00:55:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:40,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:40,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373000375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:40,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373000376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:40,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373000377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:40,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373000378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373000378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:40,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:40,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373000485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:40,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373000485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:40,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373000486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:40,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373000486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373000486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:40,673 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/253a021c104d41fba39dae34d327b40b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/253a021c104d41fba39dae34d327b40b 2024-11-12T00:55:40,676 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/A of 891b430dc04c2e04404d2006f883443e into 253a021c104d41fba39dae34d327b40b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:40,676 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:40,676 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/A, priority=13, startTime=1731372939849; duration=0sec 2024-11-12T00:55:40,676 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:40,676 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:A 2024-11-12T00:55:40,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373000695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:40,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:40,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373000695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:40,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373000696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373000696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373000696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:40,722 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/fd0600e33b3d42019556600022fe08a5 2024-11-12T00:55:40,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/185a2fb2462b4a5b86aa6e1cbeddd17f is 50, key is test_row_0/B:col10/1731372938202/Put/seqid=0 2024-11-12T00:55:40,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742389_1565 (size=12001) 2024-11-12T00:55:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T00:55:41,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373001005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:41,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:41,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373001005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373001006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373001006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:41,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373001006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:41,130 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/185a2fb2462b4a5b86aa6e1cbeddd17f 2024-11-12T00:55:41,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/c020ba64cbec4be589726b1d22be5147 is 50, key is test_row_0/C:col10/1731372938202/Put/seqid=0 2024-11-12T00:55:41,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742390_1566 (size=12001) 2024-11-12T00:55:41,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373001515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:41,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:41,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373001516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:41,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:41,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:41,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373001516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:41,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373001516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373001516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:41,538 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/c020ba64cbec4be589726b1d22be5147 2024-11-12T00:55:41,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/fd0600e33b3d42019556600022fe08a5 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/fd0600e33b3d42019556600022fe08a5 2024-11-12T00:55:41,544 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/fd0600e33b3d42019556600022fe08a5, entries=150, sequenceid=118, filesize=30.2 K 2024-11-12T00:55:41,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/185a2fb2462b4a5b86aa6e1cbeddd17f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/185a2fb2462b4a5b86aa6e1cbeddd17f 2024-11-12T00:55:41,547 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/185a2fb2462b4a5b86aa6e1cbeddd17f, entries=150, sequenceid=118, filesize=11.7 K 2024-11-12T00:55:41,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/c020ba64cbec4be589726b1d22be5147 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c020ba64cbec4be589726b1d22be5147 2024-11-12T00:55:41,551 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c020ba64cbec4be589726b1d22be5147, entries=150, sequenceid=118, filesize=11.7 K 2024-11-12T00:55:41,551 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 891b430dc04c2e04404d2006f883443e in 1645ms, sequenceid=118, compaction requested=false 2024-11-12T00:55:41,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:41,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:41,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-12T00:55:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-12T00:55:41,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-12T00:55:41,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7140 sec 2024-11-12T00:55:41,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.7190 sec 2024-11-12T00:55:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:42,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T00:55:42,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:42,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:42,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:42,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:42,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:42,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:42,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120bd36167b4284624b1d5deef41a7798f_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372940376/Put/seqid=0 2024-11-12T00:55:42,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742391_1567 (size=12304) 2024-11-12T00:55:42,537 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:42,540 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120bd36167b4284624b1d5deef41a7798f_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120bd36167b4284624b1d5deef41a7798f_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:42,540 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/2d1e100f9bc84bc2bfcf4f7eb516fcf6, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:42,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/2d1e100f9bc84bc2bfcf4f7eb516fcf6 is 175, key is test_row_0/A:col10/1731372940376/Put/seqid=0 2024-11-12T00:55:42,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373002537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:42,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742392_1568 (size=31105) 2024-11-12T00:55:42,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373002541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373002541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:42,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373002542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373002542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:42,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373002645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:42,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373002656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:42,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373002656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373002656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:42,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373002656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:42,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373002856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373002866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373002866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373002866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373002866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:42,944 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/2d1e100f9bc84bc2bfcf4f7eb516fcf6 2024-11-12T00:55:42,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/87f0d6f60b624da9ac7a200f51976e15 is 50, key is test_row_0/B:col10/1731372940376/Put/seqid=0 2024-11-12T00:55:42,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742393_1569 (size=12151) 2024-11-12T00:55:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T00:55:43,006 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:43,006 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:55:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-12T00:55:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T00:55:43,008 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:55:43,008 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:55:43,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:55:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T00:55:43,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T00:55:43,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:43,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:43,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:43,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:43,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:43,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373003165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:43,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:43,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373003176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:43,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373003176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373003176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:43,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373003176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:43,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T00:55:43,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:43,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:43,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:43,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:43,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:43,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T00:55:43,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/87f0d6f60b624da9ac7a200f51976e15 2024-11-12T00:55:43,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/69da80c0525c411ba4ab15af518be091 is 50, key is test_row_0/C:col10/1731372940376/Put/seqid=0 2024-11-12T00:55:43,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742394_1570 (size=12151) 2024-11-12T00:55:43,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T00:55:43,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:43,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:43,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:43,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:43,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:43,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:43,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T00:55:43,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:43,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:43,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:43,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:43,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T00:55:43,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373003675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:43,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:43,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373003686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373003686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:43,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373003686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:43,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373003686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:43,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/69da80c0525c411ba4ab15af518be091 2024-11-12T00:55:43,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/2d1e100f9bc84bc2bfcf4f7eb516fcf6 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/2d1e100f9bc84bc2bfcf4f7eb516fcf6 2024-11-12T00:55:43,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/2d1e100f9bc84bc2bfcf4f7eb516fcf6, entries=150, sequenceid=140, filesize=30.4 K 2024-11-12T00:55:43,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/87f0d6f60b624da9ac7a200f51976e15 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/87f0d6f60b624da9ac7a200f51976e15 2024-11-12T00:55:43,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T00:55:43,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:43,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:43,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:43,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:43,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:43,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/87f0d6f60b624da9ac7a200f51976e15, entries=150, sequenceid=140, filesize=11.9 K 2024-11-12T00:55:43,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/69da80c0525c411ba4ab15af518be091 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/69da80c0525c411ba4ab15af518be091 2024-11-12T00:55:43,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/69da80c0525c411ba4ab15af518be091, entries=150, sequenceid=140, filesize=11.9 K 2024-11-12T00:55:43,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 891b430dc04c2e04404d2006f883443e in 1251ms, sequenceid=140, compaction requested=true 2024-11-12T00:55:43,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:43,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:55:43,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:43,779 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:43,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:55:43,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:43,779 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:43,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:55:43,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:43,779 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:43,779 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/A is initiating minor compaction (all files) 2024-11-12T00:55:43,779 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:43,779 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/A in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:43,779 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/B is initiating minor compaction (all files) 2024-11-12T00:55:43,780 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/B in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:43,780 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/253a021c104d41fba39dae34d327b40b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/fd0600e33b3d42019556600022fe08a5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/2d1e100f9bc84bc2bfcf4f7eb516fcf6] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=91.0 K 2024-11-12T00:55:43,780 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/41c8d1af63344123bf8f703edc9499d8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/185a2fb2462b4a5b86aa6e1cbeddd17f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/87f0d6f60b624da9ac7a200f51976e15] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=35.5 K 2024-11-12T00:55:43,780 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:43,780 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/253a021c104d41fba39dae34d327b40b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/fd0600e33b3d42019556600022fe08a5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/2d1e100f9bc84bc2bfcf4f7eb516fcf6] 2024-11-12T00:55:43,780 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 41c8d1af63344123bf8f703edc9499d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731372937052 2024-11-12T00:55:43,780 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 253a021c104d41fba39dae34d327b40b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731372937052 2024-11-12T00:55:43,780 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 185a2fb2462b4a5b86aa6e1cbeddd17f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731372938201 2024-11-12T00:55:43,780 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd0600e33b3d42019556600022fe08a5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731372938201 2024-11-12T00:55:43,780 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 87f0d6f60b624da9ac7a200f51976e15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731372940376 2024-11-12T00:55:43,780 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d1e100f9bc84bc2bfcf4f7eb516fcf6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731372940376 2024-11-12T00:55:43,785 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:43,786 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/c84394715a2e4c0190101123de660be1 is 50, key is test_row_0/B:col10/1731372940376/Put/seqid=0 2024-11-12T00:55:43,786 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:43,788 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111268223d40fefc4110b15a22290b4bf652_891b430dc04c2e04404d2006f883443e store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:43,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742395_1571 (size=12459) 2024-11-12T00:55:43,790 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111268223d40fefc4110b15a22290b4bf652_891b430dc04c2e04404d2006f883443e, store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:43,790 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111268223d40fefc4110b15a22290b4bf652_891b430dc04c2e04404d2006f883443e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:43,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742396_1572 (size=4469) 2024-11-12T00:55:43,796 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#A#compaction#487 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:43,796 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/aa93f701f5924ace96983fe6128c2a3a is 175, key is test_row_0/A:col10/1731372940376/Put/seqid=0 2024-11-12T00:55:43,797 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/c84394715a2e4c0190101123de660be1 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c84394715a2e4c0190101123de660be1 2024-11-12T00:55:43,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742397_1573 (size=31413) 2024-11-12T00:55:43,804 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/B of 891b430dc04c2e04404d2006f883443e into c84394715a2e4c0190101123de660be1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:43,804 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:43,804 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/B, priority=13, startTime=1731372943779; duration=0sec 2024-11-12T00:55:43,804 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:43,804 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:B 2024-11-12T00:55:43,804 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:43,805 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:43,805 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/C is initiating minor compaction (all files) 2024-11-12T00:55:43,805 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/C in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:43,805 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/4779a91b267d42c694919c4e598d7898, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c020ba64cbec4be589726b1d22be5147, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/69da80c0525c411ba4ab15af518be091] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=35.5 K 2024-11-12T00:55:43,805 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 4779a91b267d42c694919c4e598d7898, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731372937052 2024-11-12T00:55:43,805 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting c020ba64cbec4be589726b1d22be5147, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731372938201 2024-11-12T00:55:43,806 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 69da80c0525c411ba4ab15af518be091, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731372940376 2024-11-12T00:55:43,811 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#C#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:43,812 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/2581f76d3f434f91853c42d12fdc6882 is 50, key is test_row_0/C:col10/1731372940376/Put/seqid=0 2024-11-12T00:55:43,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742398_1574 (size=12459) 2024-11-12T00:55:43,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T00:55:43,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:43,924 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T00:55:43,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:43,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:43,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:43,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:43,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:43,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:43,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126f2dbb959d164cbdb2461efa95867aca_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372942540/Put/seqid=0 2024-11-12T00:55:43,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742399_1575 (size=12304) 2024-11-12T00:55:43,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:43,936 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126f2dbb959d164cbdb2461efa95867aca_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126f2dbb959d164cbdb2461efa95867aca_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:43,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/b435febb9f38493ea1a314007270d26c, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:43,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/b435febb9f38493ea1a314007270d26c is 175, key is test_row_0/A:col10/1731372942540/Put/seqid=0 2024-11-12T00:55:43,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742400_1576 (size=31105) 2024-11-12T00:55:44,054 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T00:55:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T00:55:44,206 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/aa93f701f5924ace96983fe6128c2a3a as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/aa93f701f5924ace96983fe6128c2a3a 2024-11-12T00:55:44,209 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/A of 891b430dc04c2e04404d2006f883443e into aa93f701f5924ace96983fe6128c2a3a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:44,209 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:44,209 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/A, priority=13, startTime=1731372943779; duration=0sec 2024-11-12T00:55:44,209 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:44,209 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:A 2024-11-12T00:55:44,218 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/2581f76d3f434f91853c42d12fdc6882 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/2581f76d3f434f91853c42d12fdc6882 2024-11-12T00:55:44,221 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/C of 891b430dc04c2e04404d2006f883443e into 2581f76d3f434f91853c42d12fdc6882(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:44,221 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:44,221 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/C, priority=13, startTime=1731372943779; duration=0sec 2024-11-12T00:55:44,222 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:44,222 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:C 2024-11-12T00:55:44,341 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/b435febb9f38493ea1a314007270d26c 2024-11-12T00:55:44,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/d5121bcce57741d29be72195442508ca is 50, key is test_row_0/B:col10/1731372942540/Put/seqid=0 2024-11-12T00:55:44,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742401_1577 (size=12151) 2024-11-12T00:55:44,353 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/d5121bcce57741d29be72195442508ca 2024-11-12T00:55:44,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/5846a388e002453e824319e8fbf84ce1 is 50, key is test_row_0/C:col10/1731372942540/Put/seqid=0 2024-11-12T00:55:44,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742402_1578 (size=12151) 2024-11-12T00:55:44,368 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/5846a388e002453e824319e8fbf84ce1 2024-11-12T00:55:44,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/b435febb9f38493ea1a314007270d26c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/b435febb9f38493ea1a314007270d26c 2024-11-12T00:55:44,380 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/b435febb9f38493ea1a314007270d26c, entries=150, sequenceid=155, filesize=30.4 K 2024-11-12T00:55:44,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/d5121bcce57741d29be72195442508ca as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/d5121bcce57741d29be72195442508ca 2024-11-12T00:55:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,383 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/d5121bcce57741d29be72195442508ca, entries=150, sequenceid=155, filesize=11.9 K 2024-11-12T00:55:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/5846a388e002453e824319e8fbf84ce1 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/5846a388e002453e824319e8fbf84ce1 2024-11-12T00:55:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,396 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/5846a388e002453e824319e8fbf84ce1, entries=150, sequenceid=155, filesize=11.9 K 2024-11-12T00:55:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,397 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 891b430dc04c2e04404d2006f883443e in 473ms, sequenceid=155, compaction requested=false 2024-11-12T00:55:44,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:44,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:44,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-12T00:55:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-12T00:55:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-12T00:55:44,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3900 sec 2024-11-12T00:55:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.3970 sec 2024-11-12T00:55:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T00:55:44,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:44,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:44,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:44,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:44,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:44,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d2520974f9144b0bb9e46c06d0cbd0e1_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372944718/Put/seqid=0 2024-11-12T00:55:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742403_1579 (size=22268) 2024-11-12T00:55:44,762 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:44,769 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d2520974f9144b0bb9e46c06d0cbd0e1_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d2520974f9144b0bb9e46c06d0cbd0e1_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:44,773 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/79bfce959a664b02b6c13aaf77542951, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:44,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/79bfce959a664b02b6c13aaf77542951 is 175, key is test_row_0/A:col10/1731372944718/Put/seqid=0 2024-11-12T00:55:44,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731372944761, max timestamp=1731372944776Num uniques 859; 2024-11-12T00:55:44,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:55:44,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731372944761, max timestamp=1731372944776Num uniques 859; 2024-11-12T00:55:44,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:55:44,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731372944761, max timestamp=1731372944776Num uniques 859; 2024-11-12T00:55:44,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/6928b69dca1b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T00:55:44,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742404_1580 (size=65673) 2024-11-12T00:55:44,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:44,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373004799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:44,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:44,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373004799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:44,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:44,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373004802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:44,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373004805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:44,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373004806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:44,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:44,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373004916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:44,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:44,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373004916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:44,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:44,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373004916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:44,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:44,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373004916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:44,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:44,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373004917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:44,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:45,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373005125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373005125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:45,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373005126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373005126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:45,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373005135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:45,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T00:55:45,155 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:45,156 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:55:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-12T00:55:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T00:55:45,157 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:55:45,158 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:55:45,158 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:55:45,187 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/79bfce959a664b02b6c13aaf77542951 2024-11-12T00:55:45,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/f19d2d56161143f4899cf4d20afc02a4 is 50, key is test_row_0/B:col10/1731372944718/Put/seqid=0 2024-11-12T00:55:45,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742405_1581 (size=12151) 2024-11-12T00:55:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T00:55:45,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T00:55:45,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:45,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:45,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:45,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:45,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:45,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373005435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:45,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373005436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:45,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373005436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:45,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373005436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:45,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373005445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:45,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T00:55:45,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:45,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:45,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:45,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:45,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T00:55:45,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/f19d2d56161143f4899cf4d20afc02a4 2024-11-12T00:55:45,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/b1a82649727a41598163dba8646193f1 is 50, key is test_row_0/C:col10/1731372944718/Put/seqid=0 2024-11-12T00:55:45,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742406_1582 (size=12151) 2024-11-12T00:55:45,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/b1a82649727a41598163dba8646193f1 2024-11-12T00:55:45,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T00:55:45,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:45,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:45,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:45,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:45,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:45,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/79bfce959a664b02b6c13aaf77542951 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/79bfce959a664b02b6c13aaf77542951 2024-11-12T00:55:45,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/79bfce959a664b02b6c13aaf77542951, entries=350, sequenceid=167, filesize=64.1 K 2024-11-12T00:55:45,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/f19d2d56161143f4899cf4d20afc02a4 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/f19d2d56161143f4899cf4d20afc02a4 2024-11-12T00:55:45,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/f19d2d56161143f4899cf4d20afc02a4, entries=150, sequenceid=167, filesize=11.9 K 2024-11-12T00:55:45,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/b1a82649727a41598163dba8646193f1 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b1a82649727a41598163dba8646193f1 2024-11-12T00:55:45,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b1a82649727a41598163dba8646193f1, entries=150, sequenceid=167, filesize=11.9 K 2024-11-12T00:55:45,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 891b430dc04c2e04404d2006f883443e in 900ms, sequenceid=167, compaction requested=true 2024-11-12T00:55:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:55:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:55:45,631 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:45,631 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:55:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:45,632 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:45,632 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128191 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:45,632 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/B is initiating minor compaction (all files) 2024-11-12T00:55:45,632 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/B in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:45,632 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/A is initiating minor compaction (all files) 2024-11-12T00:55:45,632 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c84394715a2e4c0190101123de660be1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/d5121bcce57741d29be72195442508ca, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/f19d2d56161143f4899cf4d20afc02a4] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=35.9 K 2024-11-12T00:55:45,632 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/A in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:45,632 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/aa93f701f5924ace96983fe6128c2a3a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/b435febb9f38493ea1a314007270d26c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/79bfce959a664b02b6c13aaf77542951] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=125.2 K 2024-11-12T00:55:45,632 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:45,632 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/aa93f701f5924ace96983fe6128c2a3a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/b435febb9f38493ea1a314007270d26c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/79bfce959a664b02b6c13aaf77542951] 2024-11-12T00:55:45,632 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting c84394715a2e4c0190101123de660be1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731372940376 2024-11-12T00:55:45,633 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting d5121bcce57741d29be72195442508ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731372942539 2024-11-12T00:55:45,633 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa93f701f5924ace96983fe6128c2a3a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731372940376 2024-11-12T00:55:45,633 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting b435febb9f38493ea1a314007270d26c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731372942539 2024-11-12T00:55:45,633 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting f19d2d56161143f4899cf4d20afc02a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731372944718 2024-11-12T00:55:45,634 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79bfce959a664b02b6c13aaf77542951, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731372944711 2024-11-12T00:55:45,639 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#B#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:45,639 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/00cb1b2393ec430fb6158c47da1cbe0f is 50, key is test_row_0/B:col10/1731372944718/Put/seqid=0 2024-11-12T00:55:45,640 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:45,646 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411121ced2adc147347be88fd67240437e7a4_891b430dc04c2e04404d2006f883443e store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:45,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742407_1583 (size=12561) 2024-11-12T00:55:45,648 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411121ced2adc147347be88fd67240437e7a4_891b430dc04c2e04404d2006f883443e, store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:45,648 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121ced2adc147347be88fd67240437e7a4_891b430dc04c2e04404d2006f883443e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:45,653 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/00cb1b2393ec430fb6158c47da1cbe0f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/00cb1b2393ec430fb6158c47da1cbe0f 2024-11-12T00:55:45,657 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/B of 891b430dc04c2e04404d2006f883443e into 00cb1b2393ec430fb6158c47da1cbe0f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:45,658 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:45,658 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/B, priority=13, startTime=1731372945631; duration=0sec 2024-11-12T00:55:45,658 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:45,658 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:B 2024-11-12T00:55:45,658 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:45,660 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:45,660 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/C is initiating minor compaction (all files) 2024-11-12T00:55:45,660 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/C in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:45,660 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/2581f76d3f434f91853c42d12fdc6882, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/5846a388e002453e824319e8fbf84ce1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b1a82649727a41598163dba8646193f1] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=35.9 K 2024-11-12T00:55:45,660 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 2581f76d3f434f91853c42d12fdc6882, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731372940376 2024-11-12T00:55:45,660 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 5846a388e002453e824319e8fbf84ce1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731372942539 2024-11-12T00:55:45,661 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting b1a82649727a41598163dba8646193f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731372944718 2024-11-12T00:55:45,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742408_1584 (size=4469) 2024-11-12T00:55:45,671 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#C#compaction#497 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:45,672 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/a088148ffdda44aaab55d3e7a3925acd is 50, key is test_row_0/C:col10/1731372944718/Put/seqid=0 2024-11-12T00:55:45,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742409_1585 (size=12561) 2024-11-12T00:55:45,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T00:55:45,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:45,767 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-12T00:55:45,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:45,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:45,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:45,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:45,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:45,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T00:55:45,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112babccc7519ef4d85aa4a185a780530b8_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372944805/Put/seqid=0 2024-11-12T00:55:45,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742410_1586 (size=12304) 2024-11-12T00:55:45,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:45,779 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112babccc7519ef4d85aa4a185a780530b8_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112babccc7519ef4d85aa4a185a780530b8_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:45,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/076d170dd5b04913a3a064ac85432971, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:45,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/076d170dd5b04913a3a064ac85432971 is 175, key is test_row_0/A:col10/1731372944805/Put/seqid=0 2024-11-12T00:55:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T00:55:45,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742411_1587 (size=31105) 2024-11-12T00:55:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:45,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:45,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373005953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:45,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:45,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373005955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:45,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:45,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373005985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:45,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373005985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373005985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:46,072 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#A#compaction#496 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:46,072 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/356528941c91455fb2f846f80cd9967b is 175, key is test_row_0/A:col10/1731372944718/Put/seqid=0 2024-11-12T00:55:46,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742412_1588 (size=31515) 2024-11-12T00:55:46,080 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/a088148ffdda44aaab55d3e7a3925acd as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/a088148ffdda44aaab55d3e7a3925acd 2024-11-12T00:55:46,084 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/C of 891b430dc04c2e04404d2006f883443e into a088148ffdda44aaab55d3e7a3925acd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:46,084 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:46,084 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/C, priority=13, startTime=1731372945631; duration=0sec 2024-11-12T00:55:46,084 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:46,084 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:C 2024-11-12T00:55:46,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:46,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373006095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373006096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:46,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:46,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373006096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373006096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:46,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:46,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:46,186 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/076d170dd5b04913a3a064ac85432971 2024-11-12T00:55:46,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/7f45d9a3b0f049d69da0722774572396 is 50, key is test_row_0/B:col10/1731372944805/Put/seqid=0 2024-11-12T00:55:46,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742413_1589 (size=12151) 2024-11-12T00:55:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T00:55:46,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373006305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:46,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373006306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:46,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:46,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:46,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373006306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:46,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:46,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373006306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:46,478 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/356528941c91455fb2f846f80cd9967b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/356528941c91455fb2f846f80cd9967b 2024-11-12T00:55:46,481 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/A of 891b430dc04c2e04404d2006f883443e into 356528941c91455fb2f846f80cd9967b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:46,481 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:46,482 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/A, priority=13, startTime=1731372945631; duration=0sec 2024-11-12T00:55:46,482 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:46,482 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:A 2024-11-12T00:55:46,600 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/7f45d9a3b0f049d69da0722774572396 2024-11-12T00:55:46,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/43df6488cf5b4803bcf183572bbcb64f is 50, key is test_row_0/C:col10/1731372944805/Put/seqid=0 2024-11-12T00:55:46,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742414_1590 (size=12151) 2024-11-12T00:55:46,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373006615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:46,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:46,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373006615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373006615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:46,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373006615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:46,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:46,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373006995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:46,999 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 891b430dc04c2e04404d2006f883443e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,009 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/43df6488cf5b4803bcf183572bbcb64f 2024-11-12T00:55:47,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/076d170dd5b04913a3a064ac85432971 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/076d170dd5b04913a3a064ac85432971 2024-11-12T00:55:47,015 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/076d170dd5b04913a3a064ac85432971, entries=150, sequenceid=204, filesize=30.4 K 2024-11-12T00:55:47,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/7f45d9a3b0f049d69da0722774572396 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/7f45d9a3b0f049d69da0722774572396 2024-11-12T00:55:47,018 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/7f45d9a3b0f049d69da0722774572396, entries=150, sequenceid=204, filesize=11.9 K 2024-11-12T00:55:47,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/43df6488cf5b4803bcf183572bbcb64f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/43df6488cf5b4803bcf183572bbcb64f 2024-11-12T00:55:47,021 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/43df6488cf5b4803bcf183572bbcb64f, entries=150, sequenceid=204, filesize=11.9 K 2024-11-12T00:55:47,022 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 891b430dc04c2e04404d2006f883443e in 1255ms, sequenceid=204, compaction requested=false 2024-11-12T00:55:47,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:47,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:47,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-12T00:55:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-12T00:55:47,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-12T00:55:47,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8650 sec 2024-11-12T00:55:47,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.8690 sec 2024-11-12T00:55:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:47,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T00:55:47,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:47,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:47,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:47,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:47,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:47,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:47,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112940ec58872c042daaa75c2011ff75e36_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372947127/Put/seqid=0 2024-11-12T00:55:47,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742415_1591 (size=17284) 2024-11-12T00:55:47,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373007148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373007149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373007153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373007153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373007256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373007256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373007265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373007265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T00:55:47,306 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:47,306 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:55:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-12T00:55:47,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T00:55:47,308 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:55:47,309 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:55:47,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:55:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T00:55:47,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T00:55:47,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:47,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:47,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:47,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:47,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:47,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373007465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373007465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373007475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373007475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,538 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:47,541 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112940ec58872c042daaa75c2011ff75e36_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112940ec58872c042daaa75c2011ff75e36_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:47,542 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/c142ed622a14418f99922c57cbe7a45e, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:47,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/c142ed622a14418f99922c57cbe7a45e is 175, key is test_row_0/A:col10/1731372947127/Put/seqid=0 2024-11-12T00:55:47,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742416_1592 (size=48389) 2024-11-12T00:55:47,546 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/c142ed622a14418f99922c57cbe7a45e 2024-11-12T00:55:47,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/cad13c6bc90546329478ad79d58c3a41 is 50, key is test_row_0/B:col10/1731372947127/Put/seqid=0 2024-11-12T00:55:47,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742417_1593 (size=12151) 2024-11-12T00:55:47,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T00:55:47,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:47,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:47,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:47,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:47,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T00:55:47,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T00:55:47,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:47,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:47,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:47,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:47,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:47,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373007776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373007776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373007785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373007785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:47,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T00:55:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:47,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T00:55:47,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/cad13c6bc90546329478ad79d58c3a41 2024-11-12T00:55:47,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/63046117d9fc4c58903be9c3c5c90f79 is 50, key is test_row_0/C:col10/1731372947127/Put/seqid=0 2024-11-12T00:55:47,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742418_1594 (size=12151) 2024-11-12T00:55:48,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T00:55:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:48,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:48,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T00:55:48,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:48,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:48,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:48,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:48,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:48,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373008286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:48,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:48,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373008286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:48,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373008295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:48,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373008295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:48,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/63046117d9fc4c58903be9c3c5c90f79 2024-11-12T00:55:48,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/c142ed622a14418f99922c57cbe7a45e as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/c142ed622a14418f99922c57cbe7a45e 2024-11-12T00:55:48,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/c142ed622a14418f99922c57cbe7a45e, entries=250, sequenceid=220, filesize=47.3 K 2024-11-12T00:55:48,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/cad13c6bc90546329478ad79d58c3a41 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/cad13c6bc90546329478ad79d58c3a41 2024-11-12T00:55:48,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/cad13c6bc90546329478ad79d58c3a41, entries=150, sequenceid=220, filesize=11.9 K 2024-11-12T00:55:48,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/63046117d9fc4c58903be9c3c5c90f79 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/63046117d9fc4c58903be9c3c5c90f79 2024-11-12T00:55:48,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/63046117d9fc4c58903be9c3c5c90f79, entries=150, sequenceid=220, filesize=11.9 K 2024-11-12T00:55:48,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 891b430dc04c2e04404d2006f883443e in 1246ms, sequenceid=220, compaction requested=true 2024-11-12T00:55:48,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:48,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:55:48,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:48,374 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:48,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:55:48,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:48,374 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:48,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:55:48,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:48,375 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:48,375 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:48,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T00:55:48,375 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/A is initiating minor compaction (all files) 2024-11-12T00:55:48,375 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/B is initiating minor compaction (all files) 2024-11-12T00:55:48,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:48,375 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/A in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:48,375 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/B in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:48,375 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T00:55:48,375 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/00cb1b2393ec430fb6158c47da1cbe0f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/7f45d9a3b0f049d69da0722774572396, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/cad13c6bc90546329478ad79d58c3a41] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=36.0 K 2024-11-12T00:55:48,375 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/356528941c91455fb2f846f80cd9967b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/076d170dd5b04913a3a064ac85432971, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/c142ed622a14418f99922c57cbe7a45e] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=108.4 K 2024-11-12T00:55:48,375 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:48,375 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/356528941c91455fb2f846f80cd9967b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/076d170dd5b04913a3a064ac85432971, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/c142ed622a14418f99922c57cbe7a45e] 2024-11-12T00:55:48,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:48,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:48,376 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 00cb1b2393ec430fb6158c47da1cbe0f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731372944718 2024-11-12T00:55:48,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:48,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:48,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:48,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:48,376 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 356528941c91455fb2f846f80cd9967b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731372944718 2024-11-12T00:55:48,376 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f45d9a3b0f049d69da0722774572396, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731372944800 2024-11-12T00:55:48,376 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting cad13c6bc90546329478ad79d58c3a41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731372947127 2024-11-12T00:55:48,376 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 076d170dd5b04913a3a064ac85432971, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731372944800 2024-11-12T00:55:48,376 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting c142ed622a14418f99922c57cbe7a45e, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731372945951 2024-11-12T00:55:48,381 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:48,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111289fc08691777452aaeaee85c2459a31c_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372947152/Put/seqid=0 2024-11-12T00:55:48,383 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#B#compaction#506 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:48,383 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/cef2b0f8726b41c4a3ad98b73b6e0cae is 50, key is test_row_0/B:col10/1731372947127/Put/seqid=0 2024-11-12T00:55:48,384 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112fa9646afc259481ea79b445a4284b83e_891b430dc04c2e04404d2006f883443e store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:48,387 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112fa9646afc259481ea79b445a4284b83e_891b430dc04c2e04404d2006f883443e, store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:48,387 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fa9646afc259481ea79b445a4284b83e_891b430dc04c2e04404d2006f883443e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:48,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742419_1595 (size=12663) 2024-11-12T00:55:48,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742420_1596 (size=12304) 2024-11-12T00:55:48,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:48,394 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/cef2b0f8726b41c4a3ad98b73b6e0cae as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/cef2b0f8726b41c4a3ad98b73b6e0cae 2024-11-12T00:55:48,394 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111289fc08691777452aaeaee85c2459a31c_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111289fc08691777452aaeaee85c2459a31c_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:48,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/3613e8a1029c426c88e0a62ffbe0b270, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:48,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/3613e8a1029c426c88e0a62ffbe0b270 is 175, key is test_row_0/A:col10/1731372947152/Put/seqid=0 2024-11-12T00:55:48,398 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/B of 891b430dc04c2e04404d2006f883443e into cef2b0f8726b41c4a3ad98b73b6e0cae(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:48,398 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:48,398 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/B, priority=13, startTime=1731372948374; duration=0sec 2024-11-12T00:55:48,398 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:48,398 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:B 2024-11-12T00:55:48,398 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:48,399 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:48,399 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/C is initiating minor compaction (all files) 2024-11-12T00:55:48,399 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/C in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:48,399 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/a088148ffdda44aaab55d3e7a3925acd, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/43df6488cf5b4803bcf183572bbcb64f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/63046117d9fc4c58903be9c3c5c90f79] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=36.0 K 2024-11-12T00:55:48,400 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting a088148ffdda44aaab55d3e7a3925acd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731372944718 2024-11-12T00:55:48,400 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 43df6488cf5b4803bcf183572bbcb64f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731372944800 2024-11-12T00:55:48,401 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 63046117d9fc4c58903be9c3c5c90f79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731372947127 2024-11-12T00:55:48,419 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#C#compaction#507 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:48,419 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/c00ad81fbe2248f89f0ba77e88bd2297 is 50, key is test_row_0/C:col10/1731372947127/Put/seqid=0 2024-11-12T00:55:48,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742421_1597 (size=4469) 2024-11-12T00:55:48,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742423_1599 (size=12663) 2024-11-12T00:55:48,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742422_1598 (size=31105) 2024-11-12T00:55:48,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T00:55:48,833 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#A#compaction#505 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:48,833 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/3b3456a2ed4d4a87816941e4da51547f is 175, key is test_row_0/A:col10/1731372947127/Put/seqid=0 2024-11-12T00:55:48,836 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/c00ad81fbe2248f89f0ba77e88bd2297 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c00ad81fbe2248f89f0ba77e88bd2297 2024-11-12T00:55:48,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742424_1600 (size=31617) 2024-11-12T00:55:48,839 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/3613e8a1029c426c88e0a62ffbe0b270 2024-11-12T00:55:48,840 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/C of 891b430dc04c2e04404d2006f883443e into c00ad81fbe2248f89f0ba77e88bd2297(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:48,840 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:48,840 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/C, priority=13, startTime=1731372948374; duration=0sec 2024-11-12T00:55:48,840 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:48,840 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:C 2024-11-12T00:55:48,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/ec8b4ceda62f4c04b24bd93c18ef9bba is 50, key is test_row_0/B:col10/1731372947152/Put/seqid=0 2024-11-12T00:55:48,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742425_1601 (size=12151) 2024-11-12T00:55:48,847 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/ec8b4ceda62f4c04b24bd93c18ef9bba 2024-11-12T00:55:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/b5a72755210549b9b6e0d04fb73c4d7a is 50, key is test_row_0/C:col10/1731372947152/Put/seqid=0 2024-11-12T00:55:48,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742426_1602 (size=12151) 2024-11-12T00:55:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:49,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:49,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373009055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373009165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,240 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/3b3456a2ed4d4a87816941e4da51547f as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/3b3456a2ed4d4a87816941e4da51547f 2024-11-12T00:55:49,243 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/A of 891b430dc04c2e04404d2006f883443e into 3b3456a2ed4d4a87816941e4da51547f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:49,243 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:49,243 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/A, priority=13, startTime=1731372948374; duration=0sec 2024-11-12T00:55:49,243 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:49,243 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:A 2024-11-12T00:55:49,255 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/b5a72755210549b9b6e0d04fb73c4d7a 2024-11-12T00:55:49,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/3613e8a1029c426c88e0a62ffbe0b270 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/3613e8a1029c426c88e0a62ffbe0b270 2024-11-12T00:55:49,261 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/3613e8a1029c426c88e0a62ffbe0b270, entries=150, sequenceid=239, filesize=30.4 K 2024-11-12T00:55:49,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/ec8b4ceda62f4c04b24bd93c18ef9bba as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/ec8b4ceda62f4c04b24bd93c18ef9bba 2024-11-12T00:55:49,264 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/ec8b4ceda62f4c04b24bd93c18ef9bba, entries=150, sequenceid=239, filesize=11.9 K 2024-11-12T00:55:49,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/b5a72755210549b9b6e0d04fb73c4d7a as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b5a72755210549b9b6e0d04fb73c4d7a 2024-11-12T00:55:49,267 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b5a72755210549b9b6e0d04fb73c4d7a, entries=150, sequenceid=239, filesize=11.9 K 2024-11-12T00:55:49,268 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 891b430dc04c2e04404d2006f883443e in 893ms, sequenceid=239, compaction requested=false 2024-11-12T00:55:49,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:49,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:49,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-12T00:55:49,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-12T00:55:49,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-12T00:55:49,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9600 sec 2024-11-12T00:55:49,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.9650 sec 2024-11-12T00:55:49,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:49,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T00:55:49,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:49,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:49,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:49,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:49,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:49,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:49,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122217ee49aae94368a244b0ea770b8760_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372949054/Put/seqid=0 2024-11-12T00:55:49,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742427_1603 (size=12304) 2024-11-12T00:55:49,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373009375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373009381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373009381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373009381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373009382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T00:55:49,456 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:49,456 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T00:55:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-12T00:55:49,457 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T00:55:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T00:55:49,458 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T00:55:49,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T00:55:49,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373009496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373009496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373009496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373009496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T00:55:49,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T00:55:49,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:49,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:49,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:49,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:49,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:49,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373009696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373009705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373009705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373009705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373009705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:49,716 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:49,719 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122217ee49aae94368a244b0ea770b8760_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122217ee49aae94368a244b0ea770b8760_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:49,719 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/f8a1ca5e5b534d9898e409847b558596, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:49,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/f8a1ca5e5b534d9898e409847b558596 is 175, key is test_row_0/A:col10/1731372949054/Put/seqid=0 2024-11-12T00:55:49,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742428_1604 (size=31105) 2024-11-12T00:55:49,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T00:55:49,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:49,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:49,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:49,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:49,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T00:55:49,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T00:55:49,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:49,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:49,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:49,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:49,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373010015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:50,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:50,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373010016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:50,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:50,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373010016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373010016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:50,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T00:55:50,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:50,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T00:55:50,124 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/f8a1ca5e5b534d9898e409847b558596 2024-11-12T00:55:50,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/26a7cd6cfa2644769bf7eba6b503b502 is 50, key is test_row_0/B:col10/1731372949054/Put/seqid=0 2024-11-12T00:55:50,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742429_1605 (size=12151) 2024-11-12T00:55:50,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:50,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373010205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:50,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:50,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T00:55:50,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:50,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T00:55:50,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:50,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T00:55:50,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:50,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373010526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:50,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373010526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:50,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:50,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373010526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373010526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:50,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/26a7cd6cfa2644769bf7eba6b503b502 2024-11-12T00:55:50,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/c5336c8e42db42d99c258cdbfe603c23 is 50, key is test_row_0/C:col10/1731372949054/Put/seqid=0 2024-11-12T00:55:50,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742430_1606 (size=12151) 2024-11-12T00:55:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T00:55:50,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T00:55:50,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:50,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T00:55:50,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:50,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T00:55:50,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/c5336c8e42db42d99c258cdbfe603c23 2024-11-12T00:55:50,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/f8a1ca5e5b534d9898e409847b558596 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/f8a1ca5e5b534d9898e409847b558596 2024-11-12T00:55:50,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/f8a1ca5e5b534d9898e409847b558596, entries=150, sequenceid=256, filesize=30.4 K 2024-11-12T00:55:50,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/26a7cd6cfa2644769bf7eba6b503b502 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/26a7cd6cfa2644769bf7eba6b503b502 2024-11-12T00:55:50,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/26a7cd6cfa2644769bf7eba6b503b502, entries=150, sequenceid=256, filesize=11.9 K 2024-11-12T00:55:50,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/c5336c8e42db42d99c258cdbfe603c23 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c5336c8e42db42d99c258cdbfe603c23 2024-11-12T00:55:50,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c5336c8e42db42d99c258cdbfe603c23, entries=150, sequenceid=256, filesize=11.9 K 2024-11-12T00:55:50,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 891b430dc04c2e04404d2006f883443e in 1645ms, sequenceid=256, compaction requested=true 2024-11-12T00:55:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:55:50,952 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:55:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:50,952 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:55:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:50,953 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:50,953 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:50,953 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/A is initiating minor compaction (all files) 2024-11-12T00:55:50,953 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/B is initiating minor compaction (all files) 2024-11-12T00:55:50,953 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/A in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,953 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/B in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,953 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/3b3456a2ed4d4a87816941e4da51547f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/3613e8a1029c426c88e0a62ffbe0b270, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/f8a1ca5e5b534d9898e409847b558596] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=91.6 K 2024-11-12T00:55:50,953 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/cef2b0f8726b41c4a3ad98b73b6e0cae, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/ec8b4ceda62f4c04b24bd93c18ef9bba, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/26a7cd6cfa2644769bf7eba6b503b502] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=36.1 K 2024-11-12T00:55:50,953 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,953 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/3b3456a2ed4d4a87816941e4da51547f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/3613e8a1029c426c88e0a62ffbe0b270, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/f8a1ca5e5b534d9898e409847b558596] 2024-11-12T00:55:50,954 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b3456a2ed4d4a87816941e4da51547f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731372947127 2024-11-12T00:55:50,954 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting cef2b0f8726b41c4a3ad98b73b6e0cae, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731372947127 2024-11-12T00:55:50,954 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3613e8a1029c426c88e0a62ffbe0b270, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731372947133 2024-11-12T00:55:50,954 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting ec8b4ceda62f4c04b24bd93c18ef9bba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731372947133 2024-11-12T00:55:50,954 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8a1ca5e5b534d9898e409847b558596, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731372949017 2024-11-12T00:55:50,954 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 26a7cd6cfa2644769bf7eba6b503b502, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731372949017 2024-11-12T00:55:50,959 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:50,961 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111275a68d3857b7499b9ee04fa5283215af_891b430dc04c2e04404d2006f883443e store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:50,961 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#B#compaction#514 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:50,962 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/990cbce940fc465fbecc5be4c9b091bf is 50, key is test_row_0/B:col10/1731372949054/Put/seqid=0 2024-11-12T00:55:50,963 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111275a68d3857b7499b9ee04fa5283215af_891b430dc04c2e04404d2006f883443e, store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:50,963 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111275a68d3857b7499b9ee04fa5283215af_891b430dc04c2e04404d2006f883443e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:50,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742431_1607 (size=12765) 2024-11-12T00:55:50,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742432_1608 (size=4469) 2024-11-12T00:55:50,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33953 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T00:55:50,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:50,982 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T00:55:50,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:50,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:50,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:50,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:50,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:50,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:50,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111266bbf1736dac4962af0dc7bd8a7bc3db_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372949381/Put/seqid=0 2024-11-12T00:55:50,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742433_1609 (size=12454) 2024-11-12T00:55:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:51,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. as already flushing 2024-11-12T00:55:51,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373011261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:51,374 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/990cbce940fc465fbecc5be4c9b091bf as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/990cbce940fc465fbecc5be4c9b091bf 2024-11-12T00:55:51,376 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#A#compaction#513 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:51,376 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/4edddf43ef9d4c5d9fa0fad293f776d0 is 175, key is test_row_0/A:col10/1731372949054/Put/seqid=0 2024-11-12T00:55:51,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373011375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:51,380 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/B of 891b430dc04c2e04404d2006f883443e into 990cbce940fc465fbecc5be4c9b091bf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:51,380 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:51,380 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/B, priority=13, startTime=1731372950952; duration=0sec 2024-11-12T00:55:51,380 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:51,380 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:B 2024-11-12T00:55:51,380 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:51,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742434_1610 (size=31719) 2024-11-12T00:55:51,381 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:51,381 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/C is initiating minor compaction (all files) 2024-11-12T00:55:51,382 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/C in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:51,382 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c00ad81fbe2248f89f0ba77e88bd2297, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b5a72755210549b9b6e0d04fb73c4d7a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c5336c8e42db42d99c258cdbfe603c23] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=36.1 K 2024-11-12T00:55:51,382 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting c00ad81fbe2248f89f0ba77e88bd2297, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731372947127 2024-11-12T00:55:51,382 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting b5a72755210549b9b6e0d04fb73c4d7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731372947133 2024-11-12T00:55:51,382 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting c5336c8e42db42d99c258cdbfe603c23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731372949017 2024-11-12T00:55:51,385 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/4edddf43ef9d4c5d9fa0fad293f776d0 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/4edddf43ef9d4c5d9fa0fad293f776d0 2024-11-12T00:55:51,390 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/A of 891b430dc04c2e04404d2006f883443e into 4edddf43ef9d4c5d9fa0fad293f776d0(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:51,390 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#C#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:51,390 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:51,390 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/A, priority=13, startTime=1731372950952; duration=0sec 2024-11-12T00:55:51,390 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:51,390 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:A 2024-11-12T00:55:51,390 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/5c3ea562bbbe4b60ad5702deb5af9193 is 50, key is test_row_0/C:col10/1731372949054/Put/seqid=0 2024-11-12T00:55:51,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:51,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742435_1611 (size=12765) 2024-11-12T00:55:51,403 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111266bbf1736dac4962af0dc7bd8a7bc3db_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111266bbf1736dac4962af0dc7bd8a7bc3db_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:51,404 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/5c3ea562bbbe4b60ad5702deb5af9193 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/5c3ea562bbbe4b60ad5702deb5af9193 2024-11-12T00:55:51,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/c97d06643f944d14bfc73899b19cbade, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:51,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/c97d06643f944d14bfc73899b19cbade is 175, key is test_row_0/A:col10/1731372949381/Put/seqid=0 2024-11-12T00:55:51,410 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/C of 891b430dc04c2e04404d2006f883443e into 5c3ea562bbbe4b60ad5702deb5af9193(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:51,410 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:51,410 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/C, priority=13, startTime=1731372950952; duration=0sec 2024-11-12T00:55:51,410 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:51,410 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:C 2024-11-12T00:55:51,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742436_1612 (size=31255) 2024-11-12T00:55:51,419 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/c97d06643f944d14bfc73899b19cbade 2024-11-12T00:55:51,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/f1e1d2ee9ab045c2aa5bbf722e8dfe1b is 50, key is test_row_0/B:col10/1731372949381/Put/seqid=0 2024-11-12T00:55:51,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742437_1613 (size=12301) 2024-11-12T00:55:51,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373011536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:51,537 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 891b430dc04c2e04404d2006f883443e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:51,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:51,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373011545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:51,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373011545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:51,548 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 891b430dc04c2e04404d2006f883443e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:51,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373011545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:51,548 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 891b430dc04c2e04404d2006f883443e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:51,548 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 891b430dc04c2e04404d2006f883443e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:51,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373011586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:51,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T00:55:51,831 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/f1e1d2ee9ab045c2aa5bbf722e8dfe1b 2024-11-12T00:55:51,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/6e9e2e88fd734289902be202f69a0f01 is 50, key is test_row_0/C:col10/1731372949381/Put/seqid=0 2024-11-12T00:55:51,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742438_1614 (size=12301) 2024-11-12T00:55:51,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373011895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:52,240 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/6e9e2e88fd734289902be202f69a0f01 2024-11-12T00:55:52,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/c97d06643f944d14bfc73899b19cbade as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/c97d06643f944d14bfc73899b19cbade 2024-11-12T00:55:52,246 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/c97d06643f944d14bfc73899b19cbade, entries=150, sequenceid=274, filesize=30.5 K 2024-11-12T00:55:52,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/f1e1d2ee9ab045c2aa5bbf722e8dfe1b as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/f1e1d2ee9ab045c2aa5bbf722e8dfe1b 2024-11-12T00:55:52,249 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/f1e1d2ee9ab045c2aa5bbf722e8dfe1b, entries=150, sequenceid=274, filesize=12.0 K 2024-11-12T00:55:52,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/6e9e2e88fd734289902be202f69a0f01 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/6e9e2e88fd734289902be202f69a0f01 2024-11-12T00:55:52,252 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/6e9e2e88fd734289902be202f69a0f01, entries=150, sequenceid=274, filesize=12.0 K 2024-11-12T00:55:52,252 INFO [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 891b430dc04c2e04404d2006f883443e in 1270ms, sequenceid=274, compaction requested=false 2024-11-12T00:55:52,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:52,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:52,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/6928b69dca1b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-12T00:55:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-12T00:55:52,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-12T00:55:52,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7950 sec 2024-11-12T00:55:52,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.7990 sec 2024-11-12T00:55:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:52,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T00:55:52,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:52,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:52,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:52,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:52,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:52,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:52,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111260c840f5a2f9497f855dd615cb7e168e_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372951255/Put/seqid=0 2024-11-12T00:55:52,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742439_1615 (size=14994) 2024-11-12T00:55:52,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373012471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:52,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:52,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:52,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:52,517 INFO [Thread-2402 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2402. 2024-11-12T00:55:52,517 DEBUG [Thread-2402 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:52,517 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:52,517 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:52,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:52,518 INFO [Thread-2406 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2406. 2024-11-12T00:55:52,518 DEBUG [Thread-2406 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:52,518 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:52,518 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:52,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:52,518 INFO [Thread-2404 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2404. 2024-11-12T00:55:52,518 DEBUG [Thread-2404 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:52,518 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:52,518 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:52,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:52,521 INFO [Thread-2408 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2408. 2024-11-12T00:55:52,521 DEBUG [Thread-2408 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:52,521 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:52,521 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:52,521 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:52,522 INFO [Thread-2410 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2410. 2024-11-12T00:55:52,522 DEBUG [Thread-2410 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:52,522 DEBUG [Thread-2410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:52,522 DEBUG [Thread-2410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:52,522 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:52,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373012586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:52,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373012796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:52,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:52,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:52,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:52,818 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:52,826 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111260c840f5a2f9497f855dd615cb7e168e_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111260c840f5a2f9497f855dd615cb7e168e_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:52,828 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/b43f100f271d46038aaa8596d0abe616, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:52,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/b43f100f271d46038aaa8596d0abe616 is 175, key is test_row_0/A:col10/1731372951255/Put/seqid=0 2024-11-12T00:55:52,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742440_1616 (size=39949) 2024-11-12T00:55:53,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373013106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:53,234 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/b43f100f271d46038aaa8596d0abe616 2024-11-12T00:55:53,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/c8e754a0e1ab466db5356aaf3d63a0c1 is 50, key is test_row_0/B:col10/1731372951255/Put/seqid=0 2024-11-12T00:55:53,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742441_1617 (size=12301) 2024-11-12T00:55:53,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34856 deadline: 1731373013546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:53,548 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 891b430dc04c2e04404d2006f883443e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:53,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:53,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:53,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:53,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:53,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34808 deadline: 1731373013556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34824 deadline: 1731373013556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:53,557 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 891b430dc04c2e04404d2006f883443e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:53,557 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_2' in 891b430dc04c2e04404d2006f883443e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:53,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:53,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:53,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:53,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:53,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:53,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:53,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34822 deadline: 1731373013566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:53,566 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 6928b69dca1b,33953,1731372765559 for 'test_row_0' in 891b430dc04c2e04404d2006f883443e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T00:55:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:53,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T00:55:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34800 deadline: 1731373013616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 , the old value is region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T00:55:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., hostname=6928b69dca1b,33953,1731372765559, seqNum=5 because the exception is null or not the one we care about 2024-11-12T00:55:53,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T00:55:53,637 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:53,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/c8e754a0e1ab466db5356aaf3d63a0c1 2024-11-12T00:55:53,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/f956df0b0d9041db8f6bcc4767ea6000 is 50, key is test_row_0/C:col10/1731372951255/Put/seqid=0 2024-11-12T00:55:53,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742442_1618 (size=12301) 2024-11-12T00:55:54,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/f956df0b0d9041db8f6bcc4767ea6000 2024-11-12T00:55:54,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/b43f100f271d46038aaa8596d0abe616 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/b43f100f271d46038aaa8596d0abe616 2024-11-12T00:55:54,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/b43f100f271d46038aaa8596d0abe616, entries=200, sequenceid=292, filesize=39.0 K 2024-11-12T00:55:54,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/c8e754a0e1ab466db5356aaf3d63a0c1 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c8e754a0e1ab466db5356aaf3d63a0c1 2024-11-12T00:55:54,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c8e754a0e1ab466db5356aaf3d63a0c1, entries=150, sequenceid=292, filesize=12.0 K 2024-11-12T00:55:54,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/f956df0b0d9041db8f6bcc4767ea6000 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/f956df0b0d9041db8f6bcc4767ea6000 2024-11-12T00:55:54,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/f956df0b0d9041db8f6bcc4767ea6000, entries=150, sequenceid=292, filesize=12.0 K 2024-11-12T00:55:54,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 891b430dc04c2e04404d2006f883443e in 1671ms, sequenceid=292, compaction requested=true 2024-11-12T00:55:54,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:54,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T00:55:54,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:54,079 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:54,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T00:55:54,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:54,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 891b430dc04c2e04404d2006f883443e:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T00:55:54,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:54,079 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:54,080 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:54,080 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:54,080 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/B is initiating minor compaction (all files) 2024-11-12T00:55:54,080 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/A is initiating minor compaction (all files) 2024-11-12T00:55:54,080 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/A in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:54,080 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/B in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:54,080 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/990cbce940fc465fbecc5be4c9b091bf, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/f1e1d2ee9ab045c2aa5bbf722e8dfe1b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c8e754a0e1ab466db5356aaf3d63a0c1] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=36.5 K 2024-11-12T00:55:54,080 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/4edddf43ef9d4c5d9fa0fad293f776d0, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/c97d06643f944d14bfc73899b19cbade, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/b43f100f271d46038aaa8596d0abe616] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=100.5 K 2024-11-12T00:55:54,080 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:54,080 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. files: [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/4edddf43ef9d4c5d9fa0fad293f776d0, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/c97d06643f944d14bfc73899b19cbade, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/b43f100f271d46038aaa8596d0abe616] 2024-11-12T00:55:54,080 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 990cbce940fc465fbecc5be4c9b091bf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731372949017 2024-11-12T00:55:54,080 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4edddf43ef9d4c5d9fa0fad293f776d0, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731372949017 2024-11-12T00:55:54,080 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting f1e1d2ee9ab045c2aa5bbf722e8dfe1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731372949352 2024-11-12T00:55:54,080 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting c97d06643f944d14bfc73899b19cbade, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731372949352 2024-11-12T00:55:54,081 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting c8e754a0e1ab466db5356aaf3d63a0c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731372951255 2024-11-12T00:55:54,081 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] compactions.Compactor(225): Compacting b43f100f271d46038aaa8596d0abe616, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731372951241 2024-11-12T00:55:54,084 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:54,086 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112fc80b338257240d39607ea953adc37a0_891b430dc04c2e04404d2006f883443e store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:54,086 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#B#compaction#523 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:54,087 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/aad11ffd93694dde91f6572f31f92d9d is 50, key is test_row_0/B:col10/1731372951255/Put/seqid=0 2024-11-12T00:55:54,088 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112fc80b338257240d39607ea953adc37a0_891b430dc04c2e04404d2006f883443e, store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:54,088 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fc80b338257240d39607ea953adc37a0_891b430dc04c2e04404d2006f883443e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:54,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742443_1619 (size=13017) 2024-11-12T00:55:54,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742444_1620 (size=4469) 2024-11-12T00:55:54,495 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#A#compaction#522 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:54,496 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/2ce398a0974c404997d6f0c3a9a70f00 is 175, key is test_row_0/A:col10/1731372951255/Put/seqid=0 2024-11-12T00:55:54,501 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/aad11ffd93694dde91f6572f31f92d9d as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/aad11ffd93694dde91f6572f31f92d9d 2024-11-12T00:55:54,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742445_1621 (size=31971) 2024-11-12T00:55:54,506 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/B of 891b430dc04c2e04404d2006f883443e into aad11ffd93694dde91f6572f31f92d9d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:54,506 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:54,506 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/2ce398a0974c404997d6f0c3a9a70f00 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/2ce398a0974c404997d6f0c3a9a70f00 2024-11-12T00:55:54,506 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/B, priority=13, startTime=1731372954079; duration=0sec 2024-11-12T00:55:54,506 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T00:55:54,506 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:B 2024-11-12T00:55:54,506 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T00:55:54,507 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T00:55:54,507 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1541): 891b430dc04c2e04404d2006f883443e/C is initiating minor compaction (all files) 2024-11-12T00:55:54,507 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 891b430dc04c2e04404d2006f883443e/C in TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:54,507 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/5c3ea562bbbe4b60ad5702deb5af9193, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/6e9e2e88fd734289902be202f69a0f01, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/f956df0b0d9041db8f6bcc4767ea6000] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp, totalSize=36.5 K 2024-11-12T00:55:54,507 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c3ea562bbbe4b60ad5702deb5af9193, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731372949017 2024-11-12T00:55:54,508 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e9e2e88fd734289902be202f69a0f01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731372949352 2024-11-12T00:55:54,508 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] compactions.Compactor(225): Compacting f956df0b0d9041db8f6bcc4767ea6000, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731372951255 2024-11-12T00:55:54,509 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/A of 891b430dc04c2e04404d2006f883443e into 2ce398a0974c404997d6f0c3a9a70f00(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:54,509 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:54,509 INFO [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/A, priority=13, startTime=1731372954079; duration=0sec 2024-11-12T00:55:54,509 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:54,509 DEBUG [RS:0;6928b69dca1b:33953-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:A 2024-11-12T00:55:54,512 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 891b430dc04c2e04404d2006f883443e#C#compaction#524 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T00:55:54,513 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/ae4d61f06c9e4c6e8948d90f0430abf1 is 50, key is test_row_0/C:col10/1731372951255/Put/seqid=0 2024-11-12T00:55:54,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742446_1622 (size=13017) 2024-11-12T00:55:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33953 {}] regionserver.HRegion(8855): Flush requested on 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:54,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T00:55:54,631 INFO [Thread-2397 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2397. 2024-11-12T00:55:54,631 DEBUG [Thread-2397 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:54,631 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:54,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:54,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:54,631 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:54,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:54,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:54,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:54,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:54,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:54,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111285ed446d8b3147a18c024993472f84fb_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372952470/Put/seqid=0 2024-11-12T00:55:54,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742447_1623 (size=12454) 2024-11-12T00:55:54,926 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/ae4d61f06c9e4c6e8948d90f0430abf1 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/ae4d61f06c9e4c6e8948d90f0430abf1 2024-11-12T00:55:54,930 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 891b430dc04c2e04404d2006f883443e/C of 891b430dc04c2e04404d2006f883443e into ae4d61f06c9e4c6e8948d90f0430abf1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T00:55:54,930 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:54,930 INFO [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., storeName=891b430dc04c2e04404d2006f883443e/C, priority=13, startTime=1731372954079; duration=0sec 2024-11-12T00:55:54,930 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T00:55:54,930 DEBUG [RS:0;6928b69dca1b:33953-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 891b430dc04c2e04404d2006f883443e:C 2024-11-12T00:55:55,046 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:55,055 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111285ed446d8b3147a18c024993472f84fb_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111285ed446d8b3147a18c024993472f84fb_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:55,057 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/46431c2ed78b40dc997d32692f16ac96, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:55,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/46431c2ed78b40dc997d32692f16ac96 is 175, key is test_row_0/A:col10/1731372952470/Put/seqid=0 2024-11-12T00:55:55,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742448_1624 (size=31255) 2024-11-12T00:55:55,462 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/46431c2ed78b40dc997d32692f16ac96 2024-11-12T00:55:55,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/c4c6be2541e04dba98ccb57f0fcc1de3 is 50, key is test_row_0/B:col10/1731372952470/Put/seqid=0 2024-11-12T00:55:55,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742449_1625 (size=12301) 2024-11-12T00:55:55,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/c4c6be2541e04dba98ccb57f0fcc1de3 2024-11-12T00:55:55,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/0853627c99fb42dcb9d9befdce5ab80c is 50, key is test_row_0/C:col10/1731372952470/Put/seqid=0 2024-11-12T00:55:55,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742450_1626 (size=12301) 2024-11-12T00:55:56,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/0853627c99fb42dcb9d9befdce5ab80c 2024-11-12T00:55:56,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/46431c2ed78b40dc997d32692f16ac96 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/46431c2ed78b40dc997d32692f16ac96 2024-11-12T00:55:56,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/46431c2ed78b40dc997d32692f16ac96, entries=150, sequenceid=312, filesize=30.5 K 2024-11-12T00:55:56,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/c4c6be2541e04dba98ccb57f0fcc1de3 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c4c6be2541e04dba98ccb57f0fcc1de3 2024-11-12T00:55:56,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c4c6be2541e04dba98ccb57f0fcc1de3, entries=150, sequenceid=312, filesize=12.0 K 2024-11-12T00:55:56,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/0853627c99fb42dcb9d9befdce5ab80c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/0853627c99fb42dcb9d9befdce5ab80c 2024-11-12T00:55:56,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/0853627c99fb42dcb9d9befdce5ab80c, entries=150, sequenceid=312, filesize=12.0 K 2024-11-12T00:55:56,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 891b430dc04c2e04404d2006f883443e in 1691ms, sequenceid=312, compaction requested=false 2024-11-12T00:55:56,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 891b430dc04c2e04404d2006f883443e: 2024-11-12T00:55:57,578 INFO [Thread-2393 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2393. 2024-11-12T00:55:57,578 DEBUG [Thread-2393 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:57,578 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:57,578 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:57,579 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:57,590 INFO [Thread-2395 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2395. 2024-11-12T00:55:57,590 INFO [Thread-2399 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2399. 2024-11-12T00:55:57,590 DEBUG [Thread-2395 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:57,591 INFO [Thread-2391 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2391. 2024-11-12T00:55:57,591 DEBUG [Thread-2399 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:57,591 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:57,591 DEBUG [Thread-2399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:57,591 DEBUG [Thread-2391 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T00:55:57,591 DEBUG [Thread-2391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:57,591 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:57,591 DEBUG [Thread-2399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:57,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:57,591 DEBUG [Thread-2391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:57,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:57,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:57,591 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T00:55:57,591 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-12T00:55:57,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-12T00:55:57,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-12T00:55:57,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-12T00:55:57,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-12T00:55:57,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T00:55:57,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3242 2024-11-12T00:55:57,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3185 2024-11-12T00:55:57,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3324 2024-11-12T00:55:57,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3241 2024-11-12T00:55:57,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3177 2024-11-12T00:55:57,592 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T00:55:57,592 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T00:55:57,592 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T00:55:57,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:57,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:57,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:57,594 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T00:55:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T00:55:57,598 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372957598"}]},"ts":"1731372957598"} 2024-11-12T00:55:57,600 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T00:55:57,600 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T00:55:57,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T00:55:57,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=891b430dc04c2e04404d2006f883443e, UNASSIGN}] 2024-11-12T00:55:57,603 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=156, ppid=155, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=891b430dc04c2e04404d2006f883443e, UNASSIGN 2024-11-12T00:55:57,603 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=156 updating hbase:meta row=891b430dc04c2e04404d2006f883443e, regionState=CLOSING, regionLocation=6928b69dca1b,33953,1731372765559 2024-11-12T00:55:57,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=891b430dc04c2e04404d2006f883443e, UNASSIGN because future has completed 2024-11-12T00:55:57,605 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T00:55:57,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; CloseRegionProcedure 891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559}] 2024-11-12T00:55:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T00:55:57,758 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] handler.UnassignRegionHandler(122): Close 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:57,758 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T00:55:57,759 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1722): Closing 891b430dc04c2e04404d2006f883443e, disabling compactions & flushes 2024-11-12T00:55:57,759 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:57,759 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:57,759 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. after waiting 0 ms 2024-11-12T00:55:57,759 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:57,759 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(2902): Flushing 891b430dc04c2e04404d2006f883443e 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T00:55:57,760 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=A 2024-11-12T00:55:57,760 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:57,760 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=B 2024-11-12T00:55:57,760 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:57,760 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 891b430dc04c2e04404d2006f883443e, store=C 2024-11-12T00:55:57,761 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T00:55:57,769 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111266cb9069056d4ab68434d694052d812a_891b430dc04c2e04404d2006f883443e is 50, key is test_row_0/A:col10/1731372957587/Put/seqid=0 2024-11-12T00:55:57,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742451_1627 (size=9914) 2024-11-12T00:55:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T00:55:58,173 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T00:55:58,178 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111266cb9069056d4ab68434d694052d812a_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111266cb9069056d4ab68434d694052d812a_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:58,178 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/023327475e194da2a278fb8cbc82468c, store: [table=TestAcidGuarantees family=A region=891b430dc04c2e04404d2006f883443e] 2024-11-12T00:55:58,179 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/023327475e194da2a278fb8cbc82468c is 175, key is test_row_0/A:col10/1731372957587/Put/seqid=0 2024-11-12T00:55:58,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742452_1628 (size=22561) 2024-11-12T00:55:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T00:55:58,583 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/023327475e194da2a278fb8cbc82468c 2024-11-12T00:55:58,594 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/be750680fa87482bac46b81ab4887513 is 50, key is test_row_0/B:col10/1731372957587/Put/seqid=0 2024-11-12T00:55:58,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742453_1629 (size=9857) 2024-11-12T00:55:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T00:55:58,999 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/be750680fa87482bac46b81ab4887513 2024-11-12T00:55:59,013 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/85608c3d382f4098a3e58969457f44e0 is 50, key is test_row_0/C:col10/1731372957587/Put/seqid=0 2024-11-12T00:55:59,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742454_1630 (size=9857) 2024-11-12T00:55:59,417 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/85608c3d382f4098a3e58969457f44e0 2024-11-12T00:55:59,420 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/A/023327475e194da2a278fb8cbc82468c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/023327475e194da2a278fb8cbc82468c 2024-11-12T00:55:59,422 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/023327475e194da2a278fb8cbc82468c, entries=100, sequenceid=320, filesize=22.0 K 2024-11-12T00:55:59,423 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/B/be750680fa87482bac46b81ab4887513 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/be750680fa87482bac46b81ab4887513 2024-11-12T00:55:59,425 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/be750680fa87482bac46b81ab4887513, entries=100, sequenceid=320, filesize=9.6 K 2024-11-12T00:55:59,426 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/.tmp/C/85608c3d382f4098a3e58969457f44e0 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/85608c3d382f4098a3e58969457f44e0 2024-11-12T00:55:59,429 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/85608c3d382f4098a3e58969457f44e0, entries=100, sequenceid=320, filesize=9.6 K 2024-11-12T00:55:59,430 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 891b430dc04c2e04404d2006f883443e in 1671ms, sequenceid=320, compaction requested=true 2024-11-12T00:55:59,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/de6e8b0ec4634bf4a1c4dc9389050144, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/0f33a90008154adfa5356b595f2cca3d, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/1375d4b548d745f298b259b52854f0e4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/74909b6470004c129717c9cc5650b957, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/e62dc7277f45486bb12a25c6665856c8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/8fb09c8fe7164c1a985a221f7c983131, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/253a021c104d41fba39dae34d327b40b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/fd0600e33b3d42019556600022fe08a5, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/aa93f701f5924ace96983fe6128c2a3a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/2d1e100f9bc84bc2bfcf4f7eb516fcf6, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/b435febb9f38493ea1a314007270d26c, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/79bfce959a664b02b6c13aaf77542951, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/356528941c91455fb2f846f80cd9967b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/076d170dd5b04913a3a064ac85432971, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/c142ed622a14418f99922c57cbe7a45e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/3b3456a2ed4d4a87816941e4da51547f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/3613e8a1029c426c88e0a62ffbe0b270, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/4edddf43ef9d4c5d9fa0fad293f776d0, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/f8a1ca5e5b534d9898e409847b558596, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/c97d06643f944d14bfc73899b19cbade, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/b43f100f271d46038aaa8596d0abe616] to archive 2024-11-12T00:55:59,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:55:59,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/de6e8b0ec4634bf4a1c4dc9389050144 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/de6e8b0ec4634bf4a1c4dc9389050144 2024-11-12T00:55:59,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/0f33a90008154adfa5356b595f2cca3d to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/0f33a90008154adfa5356b595f2cca3d 2024-11-12T00:55:59,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/1375d4b548d745f298b259b52854f0e4 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/1375d4b548d745f298b259b52854f0e4 2024-11-12T00:55:59,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/74909b6470004c129717c9cc5650b957 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/74909b6470004c129717c9cc5650b957 2024-11-12T00:55:59,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/e62dc7277f45486bb12a25c6665856c8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/e62dc7277f45486bb12a25c6665856c8 2024-11-12T00:55:59,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/8fb09c8fe7164c1a985a221f7c983131 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/8fb09c8fe7164c1a985a221f7c983131 2024-11-12T00:55:59,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/253a021c104d41fba39dae34d327b40b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/253a021c104d41fba39dae34d327b40b 2024-11-12T00:55:59,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/fd0600e33b3d42019556600022fe08a5 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/fd0600e33b3d42019556600022fe08a5 2024-11-12T00:55:59,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/aa93f701f5924ace96983fe6128c2a3a to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/aa93f701f5924ace96983fe6128c2a3a 2024-11-12T00:55:59,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/2d1e100f9bc84bc2bfcf4f7eb516fcf6 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/2d1e100f9bc84bc2bfcf4f7eb516fcf6 2024-11-12T00:55:59,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/b435febb9f38493ea1a314007270d26c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/b435febb9f38493ea1a314007270d26c 2024-11-12T00:55:59,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/79bfce959a664b02b6c13aaf77542951 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/79bfce959a664b02b6c13aaf77542951 2024-11-12T00:55:59,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/356528941c91455fb2f846f80cd9967b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/356528941c91455fb2f846f80cd9967b 2024-11-12T00:55:59,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/076d170dd5b04913a3a064ac85432971 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/076d170dd5b04913a3a064ac85432971 2024-11-12T00:55:59,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/c142ed622a14418f99922c57cbe7a45e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/c142ed622a14418f99922c57cbe7a45e 2024-11-12T00:55:59,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/3b3456a2ed4d4a87816941e4da51547f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/3b3456a2ed4d4a87816941e4da51547f 2024-11-12T00:55:59,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/3613e8a1029c426c88e0a62ffbe0b270 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/3613e8a1029c426c88e0a62ffbe0b270 2024-11-12T00:55:59,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/4edddf43ef9d4c5d9fa0fad293f776d0 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/4edddf43ef9d4c5d9fa0fad293f776d0 2024-11-12T00:55:59,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/f8a1ca5e5b534d9898e409847b558596 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/f8a1ca5e5b534d9898e409847b558596 2024-11-12T00:55:59,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/c97d06643f944d14bfc73899b19cbade to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/c97d06643f944d14bfc73899b19cbade 2024-11-12T00:55:59,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/b43f100f271d46038aaa8596d0abe616 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/b43f100f271d46038aaa8596d0abe616 2024-11-12T00:55:59,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/aa5b07816228444dadad8f78f5b58214, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/7f09d0a5002e4798973e987c4abd5088, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/d3fbe87c6d394450bbc2233d83058857, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/9ed0ea5784614f8e9f2fd1dd476773dd, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/3c31b77aaa6542399398ee36c1b013a3, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/41c8d1af63344123bf8f703edc9499d8, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c6fd46b918b5466e87c7100f80ee273e, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/185a2fb2462b4a5b86aa6e1cbeddd17f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c84394715a2e4c0190101123de660be1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/87f0d6f60b624da9ac7a200f51976e15, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/d5121bcce57741d29be72195442508ca, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/00cb1b2393ec430fb6158c47da1cbe0f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/f19d2d56161143f4899cf4d20afc02a4, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/7f45d9a3b0f049d69da0722774572396, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/cef2b0f8726b41c4a3ad98b73b6e0cae, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/cad13c6bc90546329478ad79d58c3a41, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/ec8b4ceda62f4c04b24bd93c18ef9bba, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/990cbce940fc465fbecc5be4c9b091bf, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/26a7cd6cfa2644769bf7eba6b503b502, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/f1e1d2ee9ab045c2aa5bbf722e8dfe1b, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c8e754a0e1ab466db5356aaf3d63a0c1] to archive 2024-11-12T00:55:59,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:55:59,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/aa5b07816228444dadad8f78f5b58214 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/aa5b07816228444dadad8f78f5b58214 2024-11-12T00:55:59,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/7f09d0a5002e4798973e987c4abd5088 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/7f09d0a5002e4798973e987c4abd5088 2024-11-12T00:55:59,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/d3fbe87c6d394450bbc2233d83058857 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/d3fbe87c6d394450bbc2233d83058857 2024-11-12T00:55:59,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/9ed0ea5784614f8e9f2fd1dd476773dd to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/9ed0ea5784614f8e9f2fd1dd476773dd 2024-11-12T00:55:59,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/3c31b77aaa6542399398ee36c1b013a3 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/3c31b77aaa6542399398ee36c1b013a3 2024-11-12T00:55:59,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/41c8d1af63344123bf8f703edc9499d8 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/41c8d1af63344123bf8f703edc9499d8 2024-11-12T00:55:59,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c6fd46b918b5466e87c7100f80ee273e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c6fd46b918b5466e87c7100f80ee273e 2024-11-12T00:55:59,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/185a2fb2462b4a5b86aa6e1cbeddd17f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/185a2fb2462b4a5b86aa6e1cbeddd17f 2024-11-12T00:55:59,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c84394715a2e4c0190101123de660be1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c84394715a2e4c0190101123de660be1 2024-11-12T00:55:59,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/87f0d6f60b624da9ac7a200f51976e15 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/87f0d6f60b624da9ac7a200f51976e15 2024-11-12T00:55:59,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/d5121bcce57741d29be72195442508ca to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/d5121bcce57741d29be72195442508ca 2024-11-12T00:55:59,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/00cb1b2393ec430fb6158c47da1cbe0f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/00cb1b2393ec430fb6158c47da1cbe0f 2024-11-12T00:55:59,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/f19d2d56161143f4899cf4d20afc02a4 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/f19d2d56161143f4899cf4d20afc02a4 2024-11-12T00:55:59,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/7f45d9a3b0f049d69da0722774572396 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/7f45d9a3b0f049d69da0722774572396 2024-11-12T00:55:59,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/cef2b0f8726b41c4a3ad98b73b6e0cae to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/cef2b0f8726b41c4a3ad98b73b6e0cae 2024-11-12T00:55:59,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/cad13c6bc90546329478ad79d58c3a41 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/cad13c6bc90546329478ad79d58c3a41 2024-11-12T00:55:59,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/ec8b4ceda62f4c04b24bd93c18ef9bba to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/ec8b4ceda62f4c04b24bd93c18ef9bba 2024-11-12T00:55:59,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/990cbce940fc465fbecc5be4c9b091bf to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/990cbce940fc465fbecc5be4c9b091bf 2024-11-12T00:55:59,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/26a7cd6cfa2644769bf7eba6b503b502 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/26a7cd6cfa2644769bf7eba6b503b502 2024-11-12T00:55:59,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/f1e1d2ee9ab045c2aa5bbf722e8dfe1b to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/f1e1d2ee9ab045c2aa5bbf722e8dfe1b 2024-11-12T00:55:59,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c8e754a0e1ab466db5356aaf3d63a0c1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c8e754a0e1ab466db5356aaf3d63a0c1 2024-11-12T00:55:59,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/f3fb9bb36375479188fff124eb667a92, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/11f747931a42466fa9a9b98fa5a2909f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/3d27f65a8ed64410ac9d71e1dd030bd7, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b3ec7a2910624487849031d2b3994038, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/2eb41b74008f43faa1e828b159e94038, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/4779a91b267d42c694919c4e598d7898, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/6e84dcfe5f454391a3a2a5cfd05c5b05, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c020ba64cbec4be589726b1d22be5147, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/2581f76d3f434f91853c42d12fdc6882, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/69da80c0525c411ba4ab15af518be091, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/5846a388e002453e824319e8fbf84ce1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/a088148ffdda44aaab55d3e7a3925acd, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b1a82649727a41598163dba8646193f1, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/43df6488cf5b4803bcf183572bbcb64f, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c00ad81fbe2248f89f0ba77e88bd2297, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/63046117d9fc4c58903be9c3c5c90f79, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b5a72755210549b9b6e0d04fb73c4d7a, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/5c3ea562bbbe4b60ad5702deb5af9193, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c5336c8e42db42d99c258cdbfe603c23, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/6e9e2e88fd734289902be202f69a0f01, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/f956df0b0d9041db8f6bcc4767ea6000] to archive 2024-11-12T00:55:59,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T00:55:59,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/f3fb9bb36375479188fff124eb667a92 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/f3fb9bb36375479188fff124eb667a92 2024-11-12T00:55:59,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/11f747931a42466fa9a9b98fa5a2909f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/11f747931a42466fa9a9b98fa5a2909f 2024-11-12T00:55:59,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/3d27f65a8ed64410ac9d71e1dd030bd7 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/3d27f65a8ed64410ac9d71e1dd030bd7 2024-11-12T00:55:59,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b3ec7a2910624487849031d2b3994038 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b3ec7a2910624487849031d2b3994038 2024-11-12T00:55:59,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/2eb41b74008f43faa1e828b159e94038 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/2eb41b74008f43faa1e828b159e94038 2024-11-12T00:55:59,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/4779a91b267d42c694919c4e598d7898 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/4779a91b267d42c694919c4e598d7898 2024-11-12T00:55:59,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/6e84dcfe5f454391a3a2a5cfd05c5b05 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/6e84dcfe5f454391a3a2a5cfd05c5b05 2024-11-12T00:55:59,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c020ba64cbec4be589726b1d22be5147 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c020ba64cbec4be589726b1d22be5147 2024-11-12T00:55:59,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/2581f76d3f434f91853c42d12fdc6882 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/2581f76d3f434f91853c42d12fdc6882 2024-11-12T00:55:59,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/69da80c0525c411ba4ab15af518be091 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/69da80c0525c411ba4ab15af518be091 2024-11-12T00:55:59,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/5846a388e002453e824319e8fbf84ce1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/5846a388e002453e824319e8fbf84ce1 2024-11-12T00:55:59,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/a088148ffdda44aaab55d3e7a3925acd to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/a088148ffdda44aaab55d3e7a3925acd 2024-11-12T00:55:59,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b1a82649727a41598163dba8646193f1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b1a82649727a41598163dba8646193f1 2024-11-12T00:55:59,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/43df6488cf5b4803bcf183572bbcb64f to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/43df6488cf5b4803bcf183572bbcb64f 2024-11-12T00:55:59,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c00ad81fbe2248f89f0ba77e88bd2297 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c00ad81fbe2248f89f0ba77e88bd2297 2024-11-12T00:55:59,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/63046117d9fc4c58903be9c3c5c90f79 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/63046117d9fc4c58903be9c3c5c90f79 2024-11-12T00:55:59,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b5a72755210549b9b6e0d04fb73c4d7a to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/b5a72755210549b9b6e0d04fb73c4d7a 2024-11-12T00:55:59,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/5c3ea562bbbe4b60ad5702deb5af9193 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/5c3ea562bbbe4b60ad5702deb5af9193 2024-11-12T00:55:59,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c5336c8e42db42d99c258cdbfe603c23 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/c5336c8e42db42d99c258cdbfe603c23 2024-11-12T00:55:59,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/6e9e2e88fd734289902be202f69a0f01 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/6e9e2e88fd734289902be202f69a0f01 2024-11-12T00:55:59,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/f956df0b0d9041db8f6bcc4767ea6000 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/f956df0b0d9041db8f6bcc4767ea6000 2024-11-12T00:55:59,498 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/recovered.edits/323.seqid, newMaxSeqId=323, maxSeqId=4 2024-11-12T00:55:59,498 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. 2024-11-12T00:55:59,499 DEBUG [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] regionserver.HRegion(1676): Region close journal for 891b430dc04c2e04404d2006f883443e: Waiting for close lock at 1731372957759Running coprocessor pre-close hooks at 1731372957759Disabling compacts and flushes for region at 1731372957759Disabling writes for close at 1731372957759Obtaining lock to block concurrent updates at 1731372957759Preparing flush snapshotting stores in 891b430dc04c2e04404d2006f883443e at 1731372957759Finished memstore snapshotting TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731372957761 (+2 ms)Flushing stores of TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e. at 1731372957762 (+1 ms)Flushing 891b430dc04c2e04404d2006f883443e/A: creating writer at 1731372957762Flushing mob file 891b430dc04c2e04404d2006f883443e/A: appending metadata at 1731372957769 (+7 ms)Flushing mob file 891b430dc04c2e04404d2006f883443e/A: closing flushed file at 1731372957769Flushing 891b430dc04c2e04404d2006f883443e/A: appending metadata at 1731372958178 (+409 ms)Flushing 891b430dc04c2e04404d2006f883443e/A: closing flushed file at 1731372958179 (+1 ms)Flushing 891b430dc04c2e04404d2006f883443e/B: creating writer at 1731372958590 (+411 ms)Flushing 891b430dc04c2e04404d2006f883443e/B: appending metadata at 1731372958593 (+3 ms)Flushing 891b430dc04c2e04404d2006f883443e/B: closing flushed file at 1731372958593Flushing 891b430dc04c2e04404d2006f883443e/C: creating writer at 1731372959007 (+414 ms)Flushing 891b430dc04c2e04404d2006f883443e/C: appending metadata at 1731372959012 (+5 ms)Flushing 891b430dc04c2e04404d2006f883443e/C: closing flushed file at 1731372959012Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18ef480d: reopening flushed file at 1731372959419 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f11615d: reopening flushed file at 1731372959423 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@643b5860: reopening flushed file at 1731372959425 (+2 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 891b430dc04c2e04404d2006f883443e in 1671ms, sequenceid=320, compaction requested=true at 1731372959430 (+5 ms)Writing region close event to WAL at 1731372959494 (+64 ms)Running coprocessor post-close hooks at 1731372959498 (+4 ms)Closed at 1731372959498 2024-11-12T00:55:59,500 INFO [RS_CLOSE_REGION-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=157}] handler.UnassignRegionHandler(157): Closed 891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,501 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=156 updating hbase:meta row=891b430dc04c2e04404d2006f883443e, regionState=CLOSED 2024-11-12T00:55:59,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE, hasLock=false; CloseRegionProcedure 891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 because future has completed 2024-11-12T00:55:59,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-12T00:55:59,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; CloseRegionProcedure 891b430dc04c2e04404d2006f883443e, server=6928b69dca1b,33953,1731372765559 in 1.8990 sec 2024-11-12T00:55:59,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-12T00:55:59,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=891b430dc04c2e04404d2006f883443e, UNASSIGN in 1.9040 sec 2024-11-12T00:55:59,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-12T00:55:59,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9080 sec 2024-11-12T00:55:59,512 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731372959512"}]},"ts":"1731372959512"} 2024-11-12T00:55:59,514 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T00:55:59,514 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T00:55:59,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9200 sec 2024-11-12T00:55:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T00:55:59,746 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:59,747 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T00:55:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:59,752 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=158, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:59,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T00:55:59,754 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=158, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:59,758 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,761 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/recovered.edits] 2024-11-12T00:55:59,764 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/023327475e194da2a278fb8cbc82468c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/023327475e194da2a278fb8cbc82468c 2024-11-12T00:55:59,766 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/2ce398a0974c404997d6f0c3a9a70f00 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/2ce398a0974c404997d6f0c3a9a70f00 2024-11-12T00:55:59,767 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/46431c2ed78b40dc997d32692f16ac96 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/A/46431c2ed78b40dc997d32692f16ac96 2024-11-12T00:55:59,770 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/aad11ffd93694dde91f6572f31f92d9d to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/aad11ffd93694dde91f6572f31f92d9d 2024-11-12T00:55:59,772 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/be750680fa87482bac46b81ab4887513 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/be750680fa87482bac46b81ab4887513 2024-11-12T00:55:59,773 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c4c6be2541e04dba98ccb57f0fcc1de3 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/B/c4c6be2541e04dba98ccb57f0fcc1de3 2024-11-12T00:55:59,777 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/0853627c99fb42dcb9d9befdce5ab80c to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/0853627c99fb42dcb9d9befdce5ab80c 2024-11-12T00:55:59,778 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/85608c3d382f4098a3e58969457f44e0 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/85608c3d382f4098a3e58969457f44e0 2024-11-12T00:55:59,780 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/ae4d61f06c9e4c6e8948d90f0430abf1 to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/C/ae4d61f06c9e4c6e8948d90f0430abf1 2024-11-12T00:55:59,784 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/recovered.edits/323.seqid to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e/recovered.edits/323.seqid 2024-11-12T00:55:59,785 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/default/TestAcidGuarantees/891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,785 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T00:55:59,786 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T00:55:59,787 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T00:55:59,792 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120bd36167b4284624b1d5deef41a7798f_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120bd36167b4284624b1d5deef41a7798f_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,794 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122217ee49aae94368a244b0ea770b8760_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122217ee49aae94368a244b0ea770b8760_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,797 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125260167f525f4bb399fcf8009849086d_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125260167f525f4bb399fcf8009849086d_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,798 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125904766eda3345fa943c83e0a0ee0c62_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125904766eda3345fa943c83e0a0ee0c62_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,799 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111260c840f5a2f9497f855dd615cb7e168e_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111260c840f5a2f9497f855dd615cb7e168e_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,800 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111266bbf1736dac4962af0dc7bd8a7bc3db_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111266bbf1736dac4962af0dc7bd8a7bc3db_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,801 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111266cb9069056d4ab68434d694052d812a_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111266cb9069056d4ab68434d694052d812a_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,803 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126f2dbb959d164cbdb2461efa95867aca_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126f2dbb959d164cbdb2461efa95867aca_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,803 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111285ed446d8b3147a18c024993472f84fb_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111285ed446d8b3147a18c024993472f84fb_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,804 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111289fc08691777452aaeaee85c2459a31c_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111289fc08691777452aaeaee85c2459a31c_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,805 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112940ec58872c042daaa75c2011ff75e36_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112940ec58872c042daaa75c2011ff75e36_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,806 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a9eaecb81b354ddca3e7ab28bd29a24c_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a9eaecb81b354ddca3e7ab28bd29a24c_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,807 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112babccc7519ef4d85aa4a185a780530b8_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112babccc7519ef4d85aa4a185a780530b8_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,808 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c68ea44b40ba42e49a178713724745bb_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c68ea44b40ba42e49a178713724745bb_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,809 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112cd5322b389d746c295021053dba200e0_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112cd5322b389d746c295021053dba200e0_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,810 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d2520974f9144b0bb9e46c06d0cbd0e1_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d2520974f9144b0bb9e46c06d0cbd0e1_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,811 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ee7d257ed77b4c1a95266b67c623ffd8_891b430dc04c2e04404d2006f883443e to hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ee7d257ed77b4c1a95266b67c623ffd8_891b430dc04c2e04404d2006f883443e 2024-11-12T00:55:59,811 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T00:55:59,813 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=158, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:59,815 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T00:55:59,817 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T00:55:59,818 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=158, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:59,818 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T00:55:59,819 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731372959818"}]},"ts":"9223372036854775807"} 2024-11-12T00:55:59,820 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T00:55:59,821 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 891b430dc04c2e04404d2006f883443e, NAME => 'TestAcidGuarantees,,1731372929342.891b430dc04c2e04404d2006f883443e.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T00:55:59,821 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T00:55:59,821 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731372959821"}]},"ts":"9223372036854775807"} 2024-11-12T00:55:59,823 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T00:55:59,823 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=158, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T00:55:59,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 75 msec 2024-11-12T00:55:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T00:55:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T00:55:59,856 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T00:55:59,864 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 247) - Thread LEAK? -, OpenFileDescriptor=457 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=332 (was 352), ProcessCount=11 (was 11), AvailableMemoryMB=4980 (was 5056) 2024-11-12T00:55:59,864 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T00:55:59,864 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T00:55:59,864 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T00:55:59,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:59,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:59,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T00:55:59,865 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T00:55:59,865 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=491889101, stopped=false 2024-11-12T00:55:59,865 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6928b69dca1b,45925,1731372764943 2024-11-12T00:55:59,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T00:55:59,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T00:55:59,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T00:55:59,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T00:55:59,867 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T00:55:59,867 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T00:55:59,867 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T00:55:59,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:59,867 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T00:55:59,867 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T00:55:59,868 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6928b69dca1b,33953,1731372765559' ***** 2024-11-12T00:55:59,868 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T00:55:59,868 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T00:55:59,869 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T00:55:59,869 INFO [RS:0;6928b69dca1b:33953 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T00:55:59,869 INFO [RS:0;6928b69dca1b:33953 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T00:55:59,869 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(959): stopping server 6928b69dca1b,33953,1731372765559 2024-11-12T00:55:59,869 INFO [RS:0;6928b69dca1b:33953 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T00:55:59,869 INFO [RS:0;6928b69dca1b:33953 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6928b69dca1b:33953. 2024-11-12T00:55:59,869 DEBUG [RS:0;6928b69dca1b:33953 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T00:55:59,869 DEBUG [RS:0;6928b69dca1b:33953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:55:59,869 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T00:55:59,869 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T00:55:59,869 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T00:55:59,869 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T00:55:59,870 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T00:55:59,870 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-12T00:55:59,870 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T00:55:59,870 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T00:55:59,870 INFO [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T00:55:59,870 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T00:55:59,871 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T00:55:59,871 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T00:55:59,871 INFO [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-12T00:55:59,894 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/.tmp/info/2bfbff4d2d1a48f5bf50d57d1a2efecf is 95, key is TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24./info:/1731372797128/DeleteFamily/seqid=0 2024-11-12T00:55:59,894 INFO [regionserver/6928b69dca1b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T00:55:59,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742455_1631 (size=6514) 2024-11-12T00:56:00,071 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T00:56:00,220 INFO [regionserver/6928b69dca1b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T00:56:00,220 INFO [regionserver/6928b69dca1b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T00:56:00,271 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T00:56:00,297 INFO [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/.tmp/info/2bfbff4d2d1a48f5bf50d57d1a2efecf 2024-11-12T00:56:00,320 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/.tmp/ns/031641916e3c4fa1bbb32f8f2643ec68 is 93, key is TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24./ns:/1731372797080/DeleteFamily/seqid=0 2024-11-12T00:56:00,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742456_1632 (size=5906) 2024-11-12T00:56:00,471 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T00:56:00,671 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T00:56:00,724 INFO [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/.tmp/ns/031641916e3c4fa1bbb32f8f2643ec68 2024-11-12T00:56:00,741 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/.tmp/rep_barrier/173b80020f4a44d29c54dfe3cba3ddb4 is 102, key is TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24./rep_barrier:/1731372797080/DeleteFamily/seqid=0 2024-11-12T00:56:00,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742457_1633 (size=6025) 2024-11-12T00:56:00,872 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T00:56:01,072 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T00:56:01,145 INFO [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/.tmp/rep_barrier/173b80020f4a44d29c54dfe3cba3ddb4 2024-11-12T00:56:01,172 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/.tmp/table/4e53db5565c74bebaee69a47ed4d4390 is 96, key is TestAcidGuarantees,,1731372769279.8f6f5fa6d51a99bb28036e719e157e24./table:/1731372797080/DeleteFamily/seqid=0 2024-11-12T00:56:01,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742458_1634 (size=5936) 2024-11-12T00:56:01,272 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T00:56:01,472 DEBUG [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T00:56:01,576 INFO [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/.tmp/table/4e53db5565c74bebaee69a47ed4d4390 2024-11-12T00:56:01,585 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/.tmp/info/2bfbff4d2d1a48f5bf50d57d1a2efecf as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/info/2bfbff4d2d1a48f5bf50d57d1a2efecf 2024-11-12T00:56:01,588 INFO [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/info/2bfbff4d2d1a48f5bf50d57d1a2efecf, entries=12, sequenceid=90, filesize=6.4 K 2024-11-12T00:56:01,589 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/.tmp/ns/031641916e3c4fa1bbb32f8f2643ec68 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/ns/031641916e3c4fa1bbb32f8f2643ec68 2024-11-12T00:56:01,593 INFO [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/ns/031641916e3c4fa1bbb32f8f2643ec68, entries=8, sequenceid=90, filesize=5.8 K 2024-11-12T00:56:01,593 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/.tmp/rep_barrier/173b80020f4a44d29c54dfe3cba3ddb4 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/rep_barrier/173b80020f4a44d29c54dfe3cba3ddb4 2024-11-12T00:56:01,596 INFO [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/rep_barrier/173b80020f4a44d29c54dfe3cba3ddb4, entries=6, sequenceid=90, filesize=5.9 K 2024-11-12T00:56:01,597 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/.tmp/table/4e53db5565c74bebaee69a47ed4d4390 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/table/4e53db5565c74bebaee69a47ed4d4390 2024-11-12T00:56:01,599 INFO [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/table/4e53db5565c74bebaee69a47ed4d4390, entries=7, sequenceid=90, filesize=5.8 K 2024-11-12T00:56:01,600 INFO [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1729ms, sequenceid=90, compaction requested=false 2024-11-12T00:56:01,604 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-12T00:56:01,604 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T00:56:01,604 INFO [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T00:56:01,604 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731372959870Running coprocessor pre-close hooks at 1731372959870Disabling compacts and flushes for region at 1731372959870Disabling writes for close at 1731372959871 (+1 ms)Obtaining lock to block concurrent updates at 1731372959871Preparing flush snapshotting stores in 1588230740 at 1731372959871Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731372959871Flushing stores of hbase:meta,,1.1588230740 at 1731372959872 (+1 ms)Flushing 1588230740/info: creating writer at 1731372959872Flushing 1588230740/info: appending metadata at 1731372959894 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731372959894Flushing 1588230740/ns: creating writer at 1731372960302 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731372960320 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731372960320Flushing 1588230740/rep_barrier: creating writer at 1731372960727 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731372960741 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731372960741Flushing 1588230740/table: creating writer at 1731372961154 (+413 ms)Flushing 1588230740/table: appending metadata at 1731372961172 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731372961172Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23810cdd: reopening flushed file at 1731372961584 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@785a4ebf: reopening flushed file at 1731372961589 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7983deb: reopening flushed file at 1731372961593 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c5210db: reopening flushed file at 1731372961597 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1729ms, sequenceid=90, compaction requested=false at 1731372961600 (+3 ms)Writing region close event to WAL at 1731372961602 (+2 ms)Running coprocessor post-close hooks at 1731372961604 (+2 ms)Closed at 1731372961604 2024-11-12T00:56:01,604 DEBUG [RS_CLOSE_META-regionserver/6928b69dca1b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T00:56:01,673 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(976): stopping server 6928b69dca1b,33953,1731372765559; all regions closed. 2024-11-12T00:56:01,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741834_1010 (size=25344) 2024-11-12T00:56:01,684 DEBUG [RS:0;6928b69dca1b:33953 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/oldWALs 2024-11-12T00:56:01,684 INFO [RS:0;6928b69dca1b:33953 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6928b69dca1b%2C33953%2C1731372765559.meta:.meta(num 1731372768796) 2024-11-12T00:56:01,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741833_1009 (size=19757641) 2024-11-12T00:56:01,689 DEBUG [RS:0;6928b69dca1b:33953 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/oldWALs 2024-11-12T00:56:01,689 INFO [RS:0;6928b69dca1b:33953 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6928b69dca1b%2C33953%2C1731372765559:(num 1731372768315) 2024-11-12T00:56:01,689 DEBUG [RS:0;6928b69dca1b:33953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T00:56:01,689 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T00:56:01,689 INFO [RS:0;6928b69dca1b:33953 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T00:56:01,690 INFO [RS:0;6928b69dca1b:33953 {}] hbase.ChoreService(370): Chore service for: regionserver/6928b69dca1b:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T00:56:01,690 INFO [RS:0;6928b69dca1b:33953 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T00:56:01,690 INFO [regionserver/6928b69dca1b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T00:56:01,691 INFO [RS:0;6928b69dca1b:33953 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33953 2024-11-12T00:56:01,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6928b69dca1b,33953,1731372765559 2024-11-12T00:56:01,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T00:56:01,695 INFO [RS:0;6928b69dca1b:33953 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T00:56:01,696 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6928b69dca1b,33953,1731372765559] 2024-11-12T00:56:01,698 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6928b69dca1b,33953,1731372765559 already deleted, retry=false 2024-11-12T00:56:01,698 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6928b69dca1b,33953,1731372765559 expired; onlineServers=0 2024-11-12T00:56:01,698 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6928b69dca1b,45925,1731372764943' ***** 2024-11-12T00:56:01,698 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T00:56:01,698 INFO [M:0;6928b69dca1b:45925 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T00:56:01,698 INFO [M:0;6928b69dca1b:45925 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T00:56:01,699 DEBUG [M:0;6928b69dca1b:45925 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T00:56:01,699 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T00:56:01,699 DEBUG [M:0;6928b69dca1b:45925 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T00:56:01,699 DEBUG [master/6928b69dca1b:0:becomeActiveMaster-HFileCleaner.large.0-1731372767967 {}] cleaner.HFileCleaner(306): Exit Thread[master/6928b69dca1b:0:becomeActiveMaster-HFileCleaner.large.0-1731372767967,5,FailOnTimeoutGroup] 2024-11-12T00:56:01,699 DEBUG [master/6928b69dca1b:0:becomeActiveMaster-HFileCleaner.small.0-1731372767969 {}] cleaner.HFileCleaner(306): Exit Thread[master/6928b69dca1b:0:becomeActiveMaster-HFileCleaner.small.0-1731372767969,5,FailOnTimeoutGroup] 2024-11-12T00:56:01,699 INFO [M:0;6928b69dca1b:45925 {}] hbase.ChoreService(370): Chore service for: master/6928b69dca1b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T00:56:01,699 INFO [M:0;6928b69dca1b:45925 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T00:56:01,700 DEBUG [M:0;6928b69dca1b:45925 {}] master.HMaster(1795): Stopping service threads 2024-11-12T00:56:01,700 INFO [M:0;6928b69dca1b:45925 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T00:56:01,700 INFO [M:0;6928b69dca1b:45925 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T00:56:01,700 ERROR [M:0;6928b69dca1b:45925 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1228365231) connection to localhost/127.0.0.1:33743 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:33743,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-12T00:56:01,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T00:56:01,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T00:56:01,701 INFO [M:0;6928b69dca1b:45925 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T00:56:01,701 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T00:56:01,701 DEBUG [M:0;6928b69dca1b:45925 {}] zookeeper.ZKUtil(347): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T00:56:01,701 WARN [M:0;6928b69dca1b:45925 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T00:56:01,702 INFO [M:0;6928b69dca1b:45925 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/.lastflushedseqids 2024-11-12T00:56:01,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742459_1635 (size=72) 2024-11-12T00:56:01,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T00:56:01,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33953-0x100316b283a0001, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T00:56:01,799 INFO [RS:0;6928b69dca1b:33953 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T00:56:01,799 INFO [RS:0;6928b69dca1b:33953 {}] regionserver.HRegionServer(1031): Exiting; stopping=6928b69dca1b,33953,1731372765559; zookeeper connection closed. 2024-11-12T00:56:01,800 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2fc03eb6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2fc03eb6 2024-11-12T00:56:01,801 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-12T00:56:02,112 INFO [M:0;6928b69dca1b:45925 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T00:56:02,113 INFO [M:0;6928b69dca1b:45925 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T00:56:02,113 DEBUG [M:0;6928b69dca1b:45925 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T00:56:02,113 INFO [M:0;6928b69dca1b:45925 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T00:56:02,113 DEBUG [M:0;6928b69dca1b:45925 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T00:56:02,114 DEBUG [M:0;6928b69dca1b:45925 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T00:56:02,114 DEBUG [M:0;6928b69dca1b:45925 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T00:56:02,114 INFO [M:0;6928b69dca1b:45925 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=763.95 KB heapSize=935.54 KB 2024-11-12T00:56:02,138 DEBUG [M:0;6928b69dca1b:45925 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9940bb6a6830412e8f0462fc8290359c is 82, key is hbase:meta,,1/info:regioninfo/1731372768916/Put/seqid=0 2024-11-12T00:56:02,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742460_1636 (size=5672) 2024-11-12T00:56:02,542 INFO [M:0;6928b69dca1b:45925 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2126 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9940bb6a6830412e8f0462fc8290359c 2024-11-12T00:56:02,571 DEBUG [M:0;6928b69dca1b:45925 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e49cdb79783442d489eed26e56270705 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x1B/proc:d/1731372800325/Put/seqid=0 2024-11-12T00:56:02,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742461_1637 (size=43255) 2024-11-12T00:56:02,974 INFO [M:0;6928b69dca1b:45925 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=763.39 KB at sequenceid=2126 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e49cdb79783442d489eed26e56270705 2024-11-12T00:56:02,978 INFO [M:0;6928b69dca1b:45925 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e49cdb79783442d489eed26e56270705 2024-11-12T00:56:02,994 DEBUG [M:0;6928b69dca1b:45925 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f435f7cb5cb14a4390d8b07250b00bf2 is 69, key is 6928b69dca1b,33953,1731372765559/rs:state/1731372768017/Put/seqid=0 2024-11-12T00:56:02,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073742462_1638 (size=5156) 2024-11-12T00:56:03,398 INFO [M:0;6928b69dca1b:45925 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2126 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f435f7cb5cb14a4390d8b07250b00bf2 2024-11-12T00:56:03,406 DEBUG [M:0;6928b69dca1b:45925 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9940bb6a6830412e8f0462fc8290359c as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9940bb6a6830412e8f0462fc8290359c 2024-11-12T00:56:03,411 INFO [M:0;6928b69dca1b:45925 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9940bb6a6830412e8f0462fc8290359c, entries=8, sequenceid=2126, filesize=5.5 K 2024-11-12T00:56:03,412 DEBUG [M:0;6928b69dca1b:45925 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e49cdb79783442d489eed26e56270705 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e49cdb79783442d489eed26e56270705 2024-11-12T00:56:03,415 INFO [M:0;6928b69dca1b:45925 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e49cdb79783442d489eed26e56270705 2024-11-12T00:56:03,415 INFO [M:0;6928b69dca1b:45925 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e49cdb79783442d489eed26e56270705, entries=158, sequenceid=2126, filesize=42.2 K 2024-11-12T00:56:03,416 DEBUG [M:0;6928b69dca1b:45925 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f435f7cb5cb14a4390d8b07250b00bf2 as hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f435f7cb5cb14a4390d8b07250b00bf2 2024-11-12T00:56:03,419 INFO [M:0;6928b69dca1b:45925 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/21b9e606-c82f-26d5-e549-8c568b6b8ef6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f435f7cb5cb14a4390d8b07250b00bf2, entries=1, sequenceid=2126, filesize=5.0 K 2024-11-12T00:56:03,420 INFO [M:0;6928b69dca1b:45925 {}] regionserver.HRegion(3140): Finished flush of dataSize ~763.95 KB/782280, heapSize ~935.24 KB/957688, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1306ms, sequenceid=2126, compaction requested=false 2024-11-12T00:56:03,422 INFO [M:0;6928b69dca1b:45925 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T00:56:03,422 DEBUG [M:0;6928b69dca1b:45925 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731372962113Disabling compacts and flushes for region at 1731372962113Disabling writes for close at 1731372962114 (+1 ms)Obtaining lock to block concurrent updates at 1731372962114Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731372962114Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=782280, getHeapSize=957928, getOffHeapSize=0, getCellsCount=2309 at 1731372962115 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731372962116 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731372962116Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731372962138 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731372962138Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731372962551 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731372962570 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731372962570Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731372962978 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731372962994 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731372962994Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26228a48: reopening flushed file at 1731372963405 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ea9687e: reopening flushed file at 1731372963411 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24a5728d: reopening flushed file at 1731372963415 (+4 ms)Finished flush of dataSize ~763.95 KB/782280, heapSize ~935.24 KB/957688, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1306ms, sequenceid=2126, compaction requested=false at 1731372963420 (+5 ms)Writing region close event to WAL at 1731372963422 (+2 ms)Closed at 1731372963422 2024-11-12T00:56:03,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40185 is added to blk_1073741830_1006 (size=921621) 2024-11-12T00:56:03,426 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T00:56:03,426 INFO [M:0;6928b69dca1b:45925 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T00:56:03,426 INFO [M:0;6928b69dca1b:45925 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45925 2024-11-12T00:56:03,426 INFO [M:0;6928b69dca1b:45925 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T00:56:03,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T00:56:03,528 INFO [M:0;6928b69dca1b:45925 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T00:56:03,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45925-0x100316b283a0000, quorum=127.0.0.1:64666, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T00:56:03,537 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T00:56:03,540 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T00:56:03,540 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T00:56:03,541 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T00:56:03,541 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/hadoop.log.dir/,STOPPED} 2024-11-12T00:56:03,544 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T00:56:03,544 WARN [BP-2013931983-172.17.0.2-1731372762199 heartbeating to localhost/127.0.0.1:33743 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T00:56:03,544 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T00:56:03,544 WARN [BP-2013931983-172.17.0.2-1731372762199 heartbeating to localhost/127.0.0.1:33743 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2013931983-172.17.0.2-1731372762199 (Datanode Uuid 05c39f0c-41e5-44bb-b94b-55d15cfa9797) service to localhost/127.0.0.1:33743 2024-11-12T00:56:03,547 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/cluster_7cc88e89-ee61-c4b1-2bfb-37ef9ae21913/data/data1/current/BP-2013931983-172.17.0.2-1731372762199 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T00:56:03,547 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/cluster_7cc88e89-ee61-c4b1-2bfb-37ef9ae21913/data/data2/current/BP-2013931983-172.17.0.2-1731372762199 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T00:56:03,547 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T00:56:03,553 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T00:56:03,554 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T00:56:03,554 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T00:56:03,554 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T00:56:03,554 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/80e51c50-b16b-5a6b-8e39-f8557b579885/hadoop.log.dir/,STOPPED} 2024-11-12T00:56:03,569 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T00:56:03,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down